<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0,viewport-fit=cover"><title>大数据入门 | Depicter</title><meta name="author" content="fanhao"><meta name="copyright" content="fanhao"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="ffffff"><meta name="description" content="大数据入门大数据的核心工作体系 存储: 保存海量待处理的数据 计算: 完成数据的价值挖掘 传输: 协助各个环节的数据传输  大数据软件生态 数据存储    Apache Hadoop - HDFS   Apache Hadoop框架内的组件HDFS是大数据体系中使用最为广泛的分布式存储技术     Apache HBase   Apache HBase是大数据体系内使用非常广泛的NoSQL KV型">
<meta property="og:type" content="article">
<meta property="og:title" content="大数据入门">
<meta property="og:url" content="https://crappier.github.io/2023/04/13/%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%85%A5%E9%97%A8/index.html">
<meta property="og:site_name" content="Depicter">
<meta property="og:description" content="大数据入门大数据的核心工作体系 存储: 保存海量待处理的数据 计算: 完成数据的价值挖掘 传输: 协助各个环节的数据传输  大数据软件生态 数据存储    Apache Hadoop - HDFS   Apache Hadoop框架内的组件HDFS是大数据体系中使用最为广泛的分布式存储技术     Apache HBase   Apache HBase是大数据体系内使用非常广泛的NoSQL KV型">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://crappier.github.io/img/cover/cover_4.jpg">
<meta property="article:published_time" content="2023-04-13T11:28:23.000Z">
<meta property="article:modified_time" content="2023-04-22T02:25:04.241Z">
<meta property="article:author" content="fanhao">
<meta property="article:tag" content="大数据">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://crappier.github.io/img/cover/cover_4.jpg"><link rel="shortcut icon" href="/img/dinosaur.png"><link rel="canonical" href="https://crappier.github.io/2023/04/13/%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%85%A5%E9%97%A8/index.html"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox/fancybox.min.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: undefined,
  translate: undefined,
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":false,"highlightHeightLimit":false},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: true,
    post: true
  },
  runtime: '天',
  dateSuffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  lightbox: 'fancybox',
  Snackbar: undefined,
  source: {
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery/dist/fjGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/flickr-justified-gallery/dist/fjGallery.min.css'
    }
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isAnchor: false,
  percent: {
    toc: true,
    rightside: false,
  }
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '大数据入门',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2023-04-22 10:25:04'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
    win.getCSS = (url,id = false) => new Promise((resolve, reject) => {
      const link = document.createElement('link')
      link.rel = 'stylesheet'
      link.href = url
      if (id) link.id = id
      link.onerror = reject
      link.onload = link.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        link.onload = link.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(link)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', 'ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const detectApple = () => {
      if(/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><meta name="generator" content="Hexo 6.3.0"></head><body><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="/img/avatar.jpg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="sidebar-site-data site-data is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">10</div></a><a href="/tags/"><div class="headline">标签</div><div class="length-num">5</div></a><a href="/categories/"><div class="headline">分类</div><div class="length-num">0</div></a></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于笔者</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg fixed" id="page-header" style="background-image: url('/img/cover/cover_4.jpg')"><nav id="nav"><span id="blog-info"><a href="/" title="Depicter"><span class="site-name">Depicter</span></a></span><div id="menus"><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 主页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 归档</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于笔者</span></a></div></div><div id="toggle-menu"><a class="site-page" href="javascript:void(0);"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">大数据入门</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2023-04-13T11:28:23.000Z" title="发表于 2023-04-13 19:28:23">2023-04-13</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2023-04-22T02:25:04.241Z" title="更新于 2023-04-22 10:25:04">2023-04-22</time></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="大数据入门"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"><i class="fa-solid fa-spinner fa-spin"></i></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="大数据入门"><a href="#大数据入门" class="headerlink" title="大数据入门"></a>大数据入门</h1><h2 id="大数据的核心工作体系"><a href="#大数据的核心工作体系" class="headerlink" title="大数据的核心工作体系"></a>大数据的核心工作体系</h2><ul>
<li>存储: 保存海量待处理的数据</li>
<li>计算: 完成数据的价值挖掘</li>
<li>传输: 协助各个环节的数据传输</li>
</ul>
<h3 id="大数据软件生态"><a href="#大数据软件生态" class="headerlink" title="大数据软件生态"></a>大数据软件生态</h3><ul>
<li><p>数据存储</p>
<blockquote>
<p>  <strong>Apache Hadoop - HDFS</strong></p>
<p>  Apache Hadoop框架内的组件HDFS是大数据体系中使用最为广泛的分布式存储技术</p>
</blockquote>
<blockquote>
<p>  <strong>Apache HBase</strong></p>
<p>  Apache HBase是大数据体系内使用非常广泛的NoSQL KV型数据库技术, HBase是基于HDFS之上构建的</p>
</blockquote>
<blockquote>
<p>  <strong>Apache KUDU</strong></p>
<p>  Apache Kudu同样为大数据体系中使用较多的分布式存储引擎</p>
</blockquote>
<blockquote>
<p>  <strong>云平台存储组件</strong></p>
<p>  各大云平台厂商也有相应的大数据存储组件, 如阿里云的OSS, UCloud的US3, AWS的S3, 金山云的KS3等</p>
</blockquote>
</li>
<li><p>数据计算</p>
<blockquote>
<p>  <strong>Apache Hadoop - MapReduce</strong></p>
<p>  Apache Hadoop的MapReduce组件是最早一代的大数据分布式计算引擎, 对大数据的发展做出了卓越的贡献</p>
</blockquote>
<blockquote>
<p>  <strong>Apache Hive</strong></p>
<p>  Apache Hive是一款以SQL为开发语言的分布式计算框架, 底层使用了Hadoop的MapReduce技术. Apache Hive至今仍活跃在大数据一线, 被许多公司使用</p>
</blockquote>
<blockquote>
<p>  <strong>Apache Spark</strong></p>
<p>  Apache Spark是目前全球范围内最火热的分布式内存计算引擎, 是大数据体系中的明星计算产品</p>
</blockquote>
<blockquote>
<p>  <strong>Apache Flink</strong></p>
<p>  Apache Flink同样是一款明星级的大花轿分布式内存计算引擎, 特别是在实时计算(流计算)领域, Flink占据了大多数的国内市场</p>
</blockquote>
</li>
<li><p>数据传输</p>
<blockquote>
<p>  <strong>Apache Kafka</strong></p>
<p>  Apache Kafka是一款分布式消息系统, 可以完成海量规模的数据传输工作, 在大数据领域也是明星产品</p>
</blockquote>
<blockquote>
<p>  <strong>Apache Pulsar</strong></p>
<p>  Apache Pulsar同样是一款分布式的消息系统, 在大数据领域有非常多的使用者</p>
</blockquote>
<blockquote>
<p>  <strong>Apache Flume</strong></p>
<p>  Apache Flume是一款流式数据采集工具, 可以从非常多的数据源中完成数据采集传输的任务</p>
</blockquote>
<blockquote>
<p>  <strong>Apache Sqoop</strong></p>
<p>  Apache Sqoop是一款ETL工具, 可以协助大数据体系和关系型数据库之间进行数据传输</p>
</blockquote>
</li>
</ul>
<h3 id="什么是Hadoop"><a href="#什么是Hadoop" class="headerlink" title="什么是Hadoop"></a>什么是Hadoop</h3><p>Hadoop是Apache软件基金会下的顶级开源项目, 以提供:</p>
<ul>
<li>分布式数据存储</li>
<li>分布式数据计算</li>
<li>分布式资源调度</li>
</ul>
<p>为一体的整体解决方案</p>
<p>Apache Hadoop是典型的分布式软件框架, 可以部署在1台甚至上万台服务区节点上协同工作, 个人或企业可以借助Hadoop构建大规模服务区集群, 完成数据的存储和计算.</p>
<h3 id="Hadoop的发展"><a href="#Hadoop的发展" class="headerlink" title="Hadoop的发展"></a>Hadoop的发展</h3><p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230415153009.png" alt="img"></p>
<p>Hadoop是一个整体, 在其内部会细分为三个功能组件, 分别是:</p>
<ol>
<li>HDFS组件: HDFS是Hadoop内的分布式存储组件, 可以构建分布式文件系统用于数据存储</li>
<li>MapReduce组件: MapReduce是Hadoop内分布式计算组件, 提供编程接口供用户开发分布式计算程序</li>
<li>YARN: YARN是Hadoop内的分布式资源调度组件, 可以供用户调整调度大规模集群的资源使用</li>
</ol>
<h2 id="HDFS"><a href="#HDFS" class="headerlink" title="HDFS"></a>HDFS</h2><h3 id="分布式存储"><a href="#分布式存储" class="headerlink" title="分布式存储"></a>分布式存储</h3><blockquote>
<p>  为什么需要分布式存储?</p>
</blockquote>
<ul>
<li>数据量太大, 单机存储能力有上限, 需要多台服务区实现存储</li>
<li>使用分布式存储, 不仅改善了存储能力, 还对网络传输、磁盘读写、CPU、内存等方面带来了综合提升, 带来了更好的效果</li>
</ul>
<blockquote>
<p>  大数据体系中, 主要有两种分布式调度的架构模式</p>
</blockquote>
<ul>
<li>去中心化模式</li>
</ul>
<p>没有明确的中心, 多服务区之间基于<strong>特定规则</strong>进行同步协调</p>
<ul>
<li>中心化模式</li>
</ul>
<p>统一调度</p>
<h4 id="主从模式"><a href="#主从模式" class="headerlink" title="主从模式"></a>主从模式</h4><p>大数据框架中大多数都是符合<strong>中心化模式</strong></p>
<blockquote>
<p>  即: 有一个中心节点(服务器)来统筹其他服务器的工作, 统一指挥, 统一调度</p>
<p>  这种模式也被称为: <strong>一主多从模式(Master And Slaves)</strong></p>
</blockquote>
<h3 id="HDFS的基础架构"><a href="#HDFS的基础架构" class="headerlink" title="HDFS的基础架构"></a>HDFS的基础架构</h3><blockquote>
<p>  HDFS: Hadoop Distributed File System (Hadoop分布式文件系统)</p>
</blockquote>
<ul>
<li>HDFS是Hadoop技术栈内提供的分布式数据存储解决方案</li>
<li>HDFS可以在多台服务器上构建存储集群, 提供分布式数据存储能力</li>
</ul>
<blockquote>
<p>   HDFS是典型的主从模式架构</p>
</blockquote>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230413200852.png" alt="image-20230413200850496"></p>
<p><strong>NameNode</strong>(领导):</p>
<ul>
<li>HDFS系统的主角色, 是一个独立的进程</li>
<li>负责管理HDFS整个文件系统</li>
<li>负责管理DataNode</li>
</ul>
<p><strong>DataNode</strong>(员工):</p>
<ul>
<li>HDFS系统的从角色, 是一个独立进程</li>
<li>主要负责数据的存储, 及存储数据和取出数据</li>
</ul>
<p><strong>SecondaryNameNode</strong>(秘书,打杂):</p>
<ul>
<li>NameNode的辅助角色, 是一个独立进程</li>
<li>主要帮助NameNode完成元数据的整理工作</li>
</ul>
<h3 id="HDFS集群环境部署"><a href="#HDFS集群环境部署" class="headerlink" title="HDFS集群环境部署"></a>HDFS集群环境部署</h3><h4 id="VMware软件创建虚拟机服务器并创建hadoop用户"><a href="#VMware软件创建虚拟机服务器并创建hadoop用户" class="headerlink" title="VMware软件创建虚拟机服务器并创建hadoop用户"></a>VMware软件创建虚拟机服务器并创建hadoop用户</h4><p>…待更新</p>
<h4 id="在VMware虚拟机上部署HDFS集群"><a href="#在VMware虚拟机上部署HDFS集群" class="headerlink" title="在VMware虚拟机上部署HDFS集群"></a>在VMware虚拟机上部署HDFS集群</h4><p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230413204404.png" alt="image-20230413204401934"></p>
<ol>
<li><p>上传<code>Hadoop</code>安装包到<code>node1</code>节点中</p>
</li>
<li><p>解压缩安装包到<code>/export/server/</code>中</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">tar -zxvf hadoop-3.3.5.tar.gz -C /export/server</span><br></pre></td></tr></table></figure>
</li>
<li><p>构建软链接</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">cd /export/server</span><br><span class="line">ln -s /export/server/hadoop-3.3.5 hadoop</span><br></pre></td></tr></table></figure>
</li>
<li><p>进入hadoop安装包内</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">cd hadoop</span><br><span class="line">ls -l</span><br></pre></td></tr></table></figure></li>
</ol>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230413205234.png" alt="image-20230413205232893"></p>
<ul>
<li><strong>bin: 存放Hadoop的各类程序(命令)</strong></li>
<li><strong>etc: 存放Hadoop的配置文件</strong></li>
<li>include: C语言的一些头文件</li>
<li>lib: 存储Linux系统的动态链接库(.so文件)</li>
<li>libexec: 存放配置Hadoop系统的脚本文件(.sh和.cmd)</li>
<li>licenses-binary: 存放许可证文件</li>
<li><strong>sbin: 管理员程序(super bin)</strong></li>
<li>share: 存放二进制源码(Java jar包)</li>
</ul>
<ol start="5">
<li><p>修改配置文件, 应用自定义设置</p>
<blockquote>
<p>  配置HDFS集群, 主要涉及以下文件的修改:</p>
</blockquote>
<ul>
<li><p>workers</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 进入配置文件目录</span></span><br><span class="line"><span class="built_in">cd</span> /etc/hadoop</span><br><span class="line"><span class="comment"># 编辑workers文件</span></span><br><span class="line">vim workers</span><br><span class="line"><span class="comment"># 添加以下内容</span></span><br><span class="line">node1</span><br><span class="line">node2</span><br><span class="line">node3</span><br></pre></td></tr></table></figure>
</li>
<li><p>hadoop-env.sh</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 填入以下内容</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/export/server/jdk</span><br><span class="line"><span class="built_in">export</span> HADOOP_HOME=/export/server/hadoop</span><br><span class="line"><span class="built_in">export</span> HADOOP_CONF_DIR=<span class="variable">$HADOOP_HOME</span>/etc/hadoop</span><br><span class="line"><span class="built_in">export</span> HADOOP_LOG_DIR=<span class="variable">$HADOOP_HOME</span>/logs</span><br></pre></td></tr></table></figure>
</li>
<li><p>core-site.xml</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">// 在文件内部填入以下内容(替换<span class="tag">&lt;<span class="name">configuration</span>&gt;</span><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span>)</span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    	<span class="tag">&lt;<span class="name">name</span>&gt;</span>fs.defaultFS<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>hdfs://node1:8020<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">    </span><br><span class="line">    <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    	<span class="tag">&lt;<span class="name">name</span>&gt;</span>io.file.buffer.size<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>131072<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br></pre></td></tr></table></figure>
</li>
<li><p>hdfs-site.xml</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><span class="line">// 在文件内部填入以下内容(替换<span class="tag">&lt;<span class="name">configuration</span>&gt;</span><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span>)</span><br><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    	<span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.datanode.data.dir.perm<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>700<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">    </span><br><span class="line">    <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    	<span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.namenode.name.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>/data/nn<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">    </span><br><span class="line">    <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    	<span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.namenode.hosts<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>node1,node2,node3<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">    </span><br><span class="line">    <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    	<span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.blocksize<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>268435456<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">    </span><br><span class="line">    <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    	<span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.namenode.handler.count<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>100<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line">    </span><br><span class="line">    <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    	<span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.datanode.data.dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">value</span>&gt;</span>/data/dn<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br></pre></td></tr></table></figure></li>
</ul>
</li>
<li><p>准备数据目录</p>
</li>
</ol>
<ul>
<li>在node1节点:<ul>
<li><code>mkdir -p /data/nn</code></li>
<li><code>mkdir /data/dn</code></li>
</ul>
</li>
<li>在node2和node3节点:<ul>
<li><code>mkdir -p /data/dn</code></li>
</ul>
</li>
</ul>
<ol start="7">
<li>分发Hadoop文件夹</li>
</ol>
<blockquote>
<p>  将node1的hadoop安装文件远程复制到node2,node3中</p>
</blockquote>
<ul>
<li><p>分发</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">在node1中执行以下命令</span></span><br><span class="line">cd /export/server</span><br><span class="line">scp -r hadoop-3.3.5 node2:`pwd`/</span><br><span class="line">scp -r hadoop-3.3.5 node3:`pwd`/</span><br></pre></td></tr></table></figure>
</li>
<li><p>在node2中执行, 为hadoop配置软链接</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">ln -s /export/server/hadoop-3.3.5 hadoop</span><br></pre></td></tr></table></figure>
</li>
<li><p>在node3中执行, 为hadoop配置软链接</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">ln -s /export/server/hadoop-3.3.5 hadoop</span><br></pre></td></tr></table></figure></li>
</ul>
<ol start="8">
<li>配置环境变量(node1, node2, node3)</li>
</ol>
<ul>
<li><p>etc&#x2F;profile</p>
<figure class="highlight bash"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">export</span> HADOOP_HOME=/export/server/hadoop</span><br><span class="line"><span class="built_in">export</span> PATH=<span class="variable">$PATH</span>:<span class="variable">$HADOOP_HOME</span>/bin:<span class="variable">$HADOOP_HOME</span>/sbin</span><br></pre></td></tr></table></figure>
</li>
<li><p>并使用source刷新当前shell环境</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">source /etc/profile</span><br></pre></td></tr></table></figure></li>
</ul>
<ol start="9">
<li>授权为hadoop用户</li>
</ol>
<p>为确保安全, hadoop系统不以root用户启动, 而是以普通用户hadoop来启动整个hadoop服务</p>
<ul>
<li><p>以root身份, 在node1, node2, node3三台服务器上均执行以下命名</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">chown -R hadoop:hadoop /data</span><br><span class="line">chown -R hadoop:hadoop /export</span><br></pre></td></tr></table></figure></li>
</ul>
<ol start="10">
<li>格式化整个文件系统</li>
</ol>
<ul>
<li><p>格式化namenode</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">确保以hadoop用户执行</span></span><br><span class="line">su - hadoop</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">格式化namenode</span></span><br><span class="line">hadoop namenode -format</span><br></pre></td></tr></table></figure>
</li>
<li><p>启动</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">一键启动hdfs集群</span></span><br><span class="line">start-dfs.sh</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">一键关闭hdfs集群</span></span><br><span class="line">stop-dfs.sh</span><br><span class="line"></span><br></pre></td></tr></table></figure></li>
</ul>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230413220144.png" alt="image-20230413220143085"></p>
<ol start="11">
<li>查看HDFS WEBUI</li>
</ol>
<p>启动完成后, 可以在浏览器打开:</p>
<p><a target="_blank" rel="noopener" href="http://node1:9870/">http://node1:9870</a>, 可以看到hdfs文件系统的管理页面</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230413220541.png" alt="image-20230413220539293"></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230414122457.png" alt="image-20230413220623058"></p>
<h4 id="HDFS的Shell操作"><a href="#HDFS的Shell操作" class="headerlink" title="HDFS的Shell操作"></a>HDFS的Shell操作</h4><h5 id="进程启停管理"><a href="#进程启停管理" class="headerlink" title="进程启停管理"></a>进程启停管理</h5><h6 id="一键启停脚本"><a href="#一键启停脚本" class="headerlink" title="一键启停脚本"></a>一键启停脚本</h6><p>Hadoop HDFS组件内置了HDFS集群的一键启停脚本</p>
<ul>
<li><p>$HADOOP_HOME&#x2F;sbin&#x2F;start-dfs.sh, 一键启动HDFS集群</p>
<p>执行原理</p>
<ul>
<li>在执行此脚本的机器上, 启动SecondaryNameNode</li>
<li>读取core-seit.xml的内容, 确认NameNode所在机器, 并启动NameNode</li>
<li>读取workers的内容, 确认DataNode所在机器, 启动全部DataNode</li>
</ul>
</li>
<li><p>$HADOOP_HOME&#x2F;sbin&#x2F;stop-dfs.sh, 一键关闭HDFS集群</p>
<p>执行原理</p>
<ul>
<li>在执行此脚本的机器上, 关闭SecondaryNameNode</li>
<li>读取core-seit.xml的内容, 确认NameNode所在机器, 并关闭NameNode</li>
<li>读取workers的内容, 确认DataNode所在机器, 关闭全部DataNode</li>
</ul>
</li>
</ul>
<h6 id="单进程启停"><a href="#单进程启停" class="headerlink" title="单进程启停"></a>单进程启停</h6><ol>
<li><p>$HADOOP_HOME&#x2F;sbin&#x2F;hadoop-daemon.sh</p>
<p>此脚本可以单独控制<strong>所在机器</strong>的进程的启停</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hadoop-daemon.sh (start|status|stop) (namenode|secondarynamenode|datanode)</span><br></pre></td></tr></table></figure>
</li>
<li><p>$HADOOP_HOME&#x2F;bin&#x2F;hafs</p>
<p>此程序也可以用于单独控制<strong>所在机器</strong>的进程的启停</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hdfs --daemon (start|status|stop) (namenode|secondarynamenode|datanode)</span><br></pre></td></tr></table></figure></li>
</ol>
<h5 id="文件系统操作命令"><a href="#文件系统操作命令" class="headerlink" title="文件系统操作命令"></a>文件系统操作命令</h5><p>关于HDFS文件系统的操作命令, Hadoop提供了2套命令体系</p>
<ul>
<li><p>hadoop命令(老版本)</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs [generic options]</span><br></pre></td></tr></table></figure>
</li>
<li><p>hdfs命令(新版本)</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hafs dfs [generic options]</span><br></pre></td></tr></table></figure></li>
</ul>
<blockquote>
<p>  两者在文件操作系统上的用法完全一致, 使用哪一个都可以</p>
<p>  某些特殊操作需要选择hadoop命令或hdfs命令</p>
</blockquote>
<ol>
<li>创建文件夹</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -mkdir [-p] &lt;path&gt; ...</span><br><span class="line">hdfs dfs -mkdir [-p] &lt;path&gt; ...</span><br></pre></td></tr></table></figure>

<p>path 为待创建的目录</p>
<p>-p 参数与<code>Linux mkdir -p</code>相同, 会沿着路径创建父目录</p>
<ol start="2">
<li>查看指定目录下的内容</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -ls [-h] [-R] [&lt;path&gt; ...]</span><br><span class="line"> hdfs dfs -ls [-h] [-R] [&lt;path&gt; ...]</span><br></pre></td></tr></table></figure>

<blockquote>
<p>  path 为指定目录路径</p>
<p>  -h 为显示文件size</p>
<p>  -R 递归查看指定目录及其子目录</p>
<p>  <img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230414130154.png" alt="image-20230414130152554"></p>
</blockquote>
<ol start="3">
<li>上传文件到HDFS指定目录下</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -put [-f] [-p] &lt;localsrc&gt; ... &lt;dst&gt;</span><br><span class="line">hdfs dfs -put [-f] [-p] &lt;localsrc&gt; ... &lt;dst&gt;</span><br></pre></td></tr></table></figure>

<p>-f 覆盖目标文件(如果已经存在)</p>
<p>-p 保留访问和修改时间, 所有权和权限</p>
<p>localsrc 本地文件系统(客户端所在机器)</p>
<p>dst 目标文件系统</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -put word.txt /itcast</span><br><span class="line">hdfs dfs -put file:///etc/profile hdfs://node1:8020/itcast</span><br></pre></td></tr></table></figure>

<ol start="4">
<li>查看HDFS内容</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -cat &lt;src&gt; ...</span><br><span class="line">hdfs dfs -cat &lt;src&gt; ...</span><br></pre></td></tr></table></figure>

<p>读取指定文件的全部内容, 显示在标准输出控制台</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -cat /test.txt</span><br><span class="line">hdfs dfs -cat /test.txt</span><br></pre></td></tr></table></figure>

<p>读取大文件可以使用<code>|</code>配合<code>more</code></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hdfs dfs -cat &lt;src&gt; | more</span><br></pre></td></tr></table></figure>

<ol start="5">
<li>下载HDFS文件</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -get [-f] [-p] &lt;src&gt; ... &lt;localdst&gt;</span><br><span class="line">hdfs dfs -get [-f] [-p] &lt;src&gt; ... &lt;localdst&gt;</span><br></pre></td></tr></table></figure>

<p>下载文件到本地文件系统的指定目录, localdst必须是目录</p>
<blockquote>
<p>  -f 覆盖目标文件(如果已存在)</p>
<p>  -p 保留访问时间和修改时间, 所有权和权限</p>
</blockquote>
<ol start="6">
<li>拷贝HDFS文件(hdfs中的功能)</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -cp [-f] &lt;src&gt; ... &lt;dst&gt;</span><br><span class="line">hdfs dfs -cp [-f] &lt;src&gt; ... &lt;dst&gt;</span><br></pre></td></tr></table></figure>

<blockquote>
<p>  -f 覆盖目标文件(如果已存在)</p>
</blockquote>
<ol start="7">
<li>追加数据到HDFS文件中</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -appendToFile &lt;localsrc&gt; ... &lt;dst&gt;</span><br><span class="line">hdfs dfs -appendToFile &lt;localsrc&gt; ... &lt;dst&gt;</span><br></pre></td></tr></table></figure>

<p>将所有给定的本地文件的内容追加到给定的<code>dst</code>文件中</p>
<p><code>dst</code>文件如果不存在, 则创建该文件</p>
<p>如果<code>localsrc</code>为<code>-</code>, 则输入为从标准输入中读取</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hdfs dfs -appendToFile append.txt /test.txt</span><br></pre></td></tr></table></figure>

<ol start="8">
<li>HDFS数据移动操作</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -mv &lt;src&gt; ... &lt;dst&gt;</span><br><span class="line">hdfs dfs -mv &lt;src&gt; ... &lt;dst&gt;</span><br></pre></td></tr></table></figure>

<p>移动文件到指定文件夹下</p>
<p>可以使用该命令移动数据, 重命名文件的名称</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hdfs dfs -mv /test.txt /home/test_2.txt</span><br></pre></td></tr></table></figure>

<ol start="9">
<li>HDFS数据删除操作</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -rm -r [-skipTrash] URI [URI ...]</span><br><span class="line">hdfs dfs -rm -r [-skipTrash] URI [URI ...]</span><br></pre></td></tr></table></figure>

<p>删除指定路径的文件或文件夹</p>
<p><code>-skipTrash</code> 跳过回收站, 直接删除</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">hdfs dfs -rm -r /home</span><br><span class="line">hdfs dfs -rm /work/123.txt</span><br></pre></td></tr></table></figure>

<blockquote>
<p>  <strong>回收站功能默认关闭, 如果要开启需要在core-site.xml中配置</strong></p>
</blockquote>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>fs.trash.interval<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">   <span class="tag">&lt;<span class="name">value</span>&gt;</span>1440<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>fs.trash.checkpoint.interval<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">   <span class="tag">&lt;<span class="name">value</span>&gt;</span>120<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<blockquote>
<p>  回收站的默认位置在: &#x2F;user&#x2F;hadoop(用户名)&#x2F;.Trash</p>
</blockquote>
<p>更多其他命令参考: <a target="_blank" rel="noopener" href="https://hadoop.apache.org/docs/r3.3.5/hadoop-project-dist/hadoop-common/FileSystemShell.html">https://hadoop.apache.org/docs/r3.3.5/hadoop-project-dist/hadoop-common/FileSystemShell.html</a></p>
<h5 id="HDFS-WEB浏览"><a href="#HDFS-WEB浏览" class="headerlink" title="HDFS WEB浏览"></a>HDFS WEB浏览</h5><p>除了使用命令操作HDFS文件系统以外, 在HDFS的WEB UI上也可以查看HDFS文件系统的内容</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230414175041.png" alt="image-20230414175040311"></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230414175318.png" alt="image-20230414175316562"></p>
<p>在使用WEB浏览器操作文件系统时, 一般会遇到权限问题</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230414175416.png" alt="image-20230414175414658"></p>
<p>由于WEB浏览器中是以匿名用户<code>dr.who</code>登录的, 只具有只读权限</p>
<p>如果需要以特权用户在浏览器中进行操作, 可以在<code>core-site.xml</code>中配置并重启集群</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>hadoop.http.staticuser.user<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>hadoop<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<blockquote>
<p>  <font color=Crimson>但是不建议这样做</font></p>
<ul>
<li><font color=Crimson>WEB UI只需要低级权限即可</font></li>
<li><font color=Crimson>以防数据泄露和丢失</font></li>
</ul>
</blockquote>
<h5 id="HDFS客户端-Jetbrains产品插件"><a href="#HDFS客户端-Jetbrains产品插件" class="headerlink" title="HDFS客户端-Jetbrains产品插件"></a>HDFS客户端-Jetbrains产品插件</h5><p><strong>Big Data Tools插件</strong></p>
<p>在Jetbrains的产品中, 均可以安装插件, 其中: <font color="blue">Big Data Tools</font>插件可以帮助我们方便地操作HDFS, 如:</p>
<ul>
<li>Intellij IDEA</li>
<li>PyCharm</li>
<li>DataGrip</li>
</ul>
<ol>
<li>此处使用<font color='pink'>PyCharm</font>进行相关操作</li>
</ol>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230414201032.png" alt="image-20230414201030544"></p>
<p>在设置-&gt;插件-&gt;Marketplace下载<strong>Big Data Tools</strong>, 安装并重启IDE</p>
<ol start="2">
<li>配置windows</li>
</ol>
<ul>
<li>解压Hadoop安装包, eg:<code>D:/hadoop3.3.5</code></li>
<li>设置<code>$HADOOP_HOME</code>环境变量: <code>D:/hadoop3.3.5</code></li>
</ul>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230414202250.png" alt="image-20230414202247320"></p>
<p>在PyCharm右侧边栏找到<strong>大数据工具</strong>选择<strong>HDFS</strong>, 测试链接<font color='red'>如果报错, 则重启电脑</font></p>
<p><img src="C:\Users\Crappy\AppData\Roaming\Typora\typora-user-images\image-20230414204034916.png" alt="image-20230414204034916"></p>
<h4 id="HDFS存储原理"><a href="#HDFS存储原理" class="headerlink" title="HDFS存储原理"></a>HDFS存储原理</h4><h5 id="数据如何在HDFS中存储的"><a href="#数据如何在HDFS中存储的" class="headerlink" title="数据如何在HDFS中存储的"></a>数据如何在HDFS中存储的</h5><blockquote>
<p>  分布式存储: 每个服务器(节点)存储文件的一部分</p>
</blockquote>
<p><font color='red'>问题1: 文件大小不一样, 不利于管理</font></p>
<p><font color='blue'>解决: 设定统一的管理单位 <font size='4'>block块</font></font></p>
<p>Block块是HDFS的最小存储单位, 每个256MB(可以在配置文件进行修改)</p>
<h5 id="数据在HDFS中如何保证安全的"><a href="#数据在HDFS中如何保证安全的" class="headerlink" title="数据在HDFS中如何保证安全的"></a>数据在HDFS中如何保证安全的</h5><p><font color='red'>问题2: 块可能丢失, 数据安全性低</font></p>
<p><font color='blue'>解决: 将block块创建多个备份, 复制到其他服务器上, 数据冗余处理, 提供安全性</font></p>
<h5 id="HDFS副本块数量的配置"><a href="#HDFS副本块数量的配置" class="headerlink" title="HDFS副本块数量的配置"></a>HDFS副本块数量的配置</h5><ul>
<li>在<font color='deep-pink'>hdfs-site.xml</font>中配置如下属性</li>
</ul>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.replication<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>3<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<p>此属性的默认值为<code>3</code>, 一般情况下无需主动配置, 如果需要自定义这个属性, 需要对每一台服务器的<font color='deep-pink'>hdfs-site.xml</font>文件进行设置.</p>
<ul>
<li>在上传文件时, 可以临时决定被上传文件以多少个副本存储</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -D dfs.replication=2 -put test.txt /tmp/</span><br></pre></td></tr></table></figure>

<p>上述命令临时设置上传test.txt文件的副本数为2</p>
<ul>
<li>对于已存在的HDFS文件, 修改dfs.replication属性不会生效, 如果需要修改已存在文件可以通过以下命令</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hadoop fs -setrep [-R] 2 path</span><br></pre></td></tr></table></figure>

<blockquote>
<p>  指定path的内容会被修改为2个副本存储</p>
<p>  -R 表示对子目录也生效</p>
</blockquote>
<ul>
<li>fsck命令检查文件的副本数</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hdfs fsck path [-files [-blocks [-locations]]]</span><br></pre></td></tr></table></figure>

<p>fsck命令可以检查指定路径是否正常</p>
<ul>
<li><ul>
<li><code>-files</code> 可以列出路径内的文件状态</li>
<li><code>-files -blocks</code> 输出文件块报告(有几个块, 多少副本)</li>
<li><code>-flies -blocks -locations</code> 输出每一个block的详情</li>
</ul>
</li>
</ul>
<p>对于块(block), hdfs默认设置为256MB一个, 也就是1GB会被划分为4个block存储</p>
<p>对块的大小设置可以通过以下配置:</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>dfs.blocksize<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>268435456<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">description</span>&gt;</span>设置HDFS块大小, 256*1024*1024b=256MB<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h5 id="NameNode元数据"><a href="#NameNode元数据" class="headerlink" title="NameNode元数据"></a>NameNode元数据</h5><p>NameNode基于一批edits和一个fsimage文件的配合完成整个文件系统的管理和维护</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230414221039.png" alt="image-20230414221038877"></p>
<ul>
<li>edits</li>
</ul>
<p>流水账文件, 记录了HDFS中的每一次操作, 以及本次操作影响的文件对应的block</p>
<p><font color='deep-pink'>存在多个edits文件, 确保不会存在超大的edits文件, 保证检索性能</font></p>
<ul>
<li>fsimage</li>
</ul>
<p>多个edits文件的合并(最终结果), 生成一个fsimage文件</p>
<ul>
<li>NameNode元数据管理维护</li>
</ul>
<p>NameNode基于edits和fsimage的配合, 完成整个文件系统的文件管理</p>
<ol>
<li>每次对HDFS的操作均被edits文件记录</li>
<li>edits文件达到大小上限时, 开启新的edits记录</li>
<li>定期进行edits文件的合并<ol>
<li>如果当前fsimage不存在, 则将全部edits合并为第一个fsimage</li>
<li>如果已存在fsimage文件, 则将全部的edits和已存在的fsimage合并为新的fsimage</li>
</ol>
</li>
<li>重复以上流程</li>
</ol>
<ul>
<li>元数据合并控制参数</li>
</ul>
<p>对于元数据的合并, 是一个定时过程, 基于:</p>
<ol>
<li><code>dfs.namenode.checkpoint.period</code>, 默认3600s</li>
<li><code>dfs.namenode.checkpoint.txns</code>, 默认100w次事务</li>
</ol>
<p>只要有一个达成条件就执行合并(默认60s检查一次)</p>
<ul>
<li><ul>
<li><code>dfs.namenode.checkpoint.check.period</code></li>
</ul>
</li>
</ul>
<p><strong><font color='cred'>SecondaryNameNode</font>的作用</strong></p>
<p>SecondatyNameNode会通过http从NameNode拉取数据(edits和fsimage), 合并完成后提供给NameNode使用</p>
<h5 id="HDFS数据的读写流程"><a href="#HDFS数据的读写流程" class="headerlink" title="HDFS数据的读写流程"></a>HDFS数据的读写流程</h5><ul>
<li>数据写入流程</li>
</ul>
<ol>
<li>客户端向NameNode发起请求</li>
<li>NameNode审核权限, 剩余空间后, 满足条件允许写入, 并告知客户端写入的DataNode地址</li>
<li>客户端向指定的DataNode发送数据包</li>
<li>被写入数据的DataNode同时完成数据副本的复制工作, 将其接收的数据分发给其他DataNode</li>
<li>DataNode1复制给DataNode2, 然后基于DataNode2复制给DataNode3和DataNode4…</li>
<li>写入完成客户端通知NameNode, NameNode做元数据记录工作</li>
</ol>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230414225411.png" alt="image-20230414225409686"></p>
<p>关键点:</p>
<p>a. <font color='cpink'>NameNode不负责数据写入</font>, 只负责元数据记录的权限审批</p>
<p>b. 客户端直接向<font color='cpink'>1台DataNode写数据</font>, 这个DataNode一般是<font color=cpink>离客户端最近(网络距离)</font>的那个</p>
<p>c. 数据快副本的复制工作, <font color='cpink'>由DataNode之间自行完成</font>(构建一个PipLine, 按顺序复制分发)</p>
<ul>
<li>数据读取流程</li>
</ul>
<ol>
<li>客户端向NameNode申请读取某文件</li>
<li>NameNode判断客户端权限等信息后, 允许读取, 并返回此文件的block列表</li>
<li>客户端拿到block列表后自行寻找DataNode读取</li>
</ol>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230414230530.png" alt="image-20230414230525534"></p>
<p>关键点:</p>
<p>a. 数据同样不通过NameNode提供</p>
<p>b. NameNode提供的block列表, 会基于网络距离计算, 提供离客户端最近的, 因为1个block有3份, 会尽量找离客户端最近的那一份让其读取</p>
<h2 id="分布式计算"><a href="#分布式计算" class="headerlink" title="分布式计算"></a>分布式计算</h2><h3 id="分布式计算概述"><a href="#分布式计算概述" class="headerlink" title="分布式计算概述"></a>分布式计算概述</h3><blockquote>
<p>  计算: 基于数据分析得到结论</p>
</blockquote>
<h4 id="分散-gt-汇总模式-MapReduce"><a href="#分散-gt-汇总模式-MapReduce" class="headerlink" title="分散-&gt;汇总模式(MapReduce)"></a>分散-&gt;汇总模式(MapReduce)</h4><ul>
<li>将数据分片, 多台服务器各自负责一部分数据处理</li>
<li>然后将各自的结果进行汇总处理</li>
<li>得到最终想要的计算结果</li>
</ul>
<h4 id="中心调度-gt-步骤执行模式-Spark-Flink"><a href="#中心调度-gt-步骤执行模式-Spark-Flink" class="headerlink" title="中心调度-&gt;步骤执行模式(Spark, Flink)"></a>中心调度-&gt;步骤执行模式(Spark, Flink)</h4><ul>
<li>由一个节点作为中心调度管理者</li>
<li>将任务划分为几个具体步骤</li>
<li>管理者安排每个机器执行任务</li>
<li>最终得到结果</li>
</ul>
<h3 id="MapReduce概述"><a href="#MapReduce概述" class="headerlink" title="MapReduce概述"></a>MapReduce概述</h3><blockquote>
<p>  MapReduce是Hadoop内提供的进行分布式计算的组件</p>
</blockquote>
<p>MapReduce是”分散-&gt;汇总”模式的分布式计算框架, 可供开发人员开发相关程序进行分布式数据计算.</p>
<p>MapReduce提供了2个编程接口:</p>
<ul>
<li>Map: 映射<ul>
<li>提供了”分散”的功能, 由服务器分布式对数据进行处理</li>
</ul>
</li>
<li>Reduce: 聚合(归纳)<ul>
<li>提供了”汇总(聚合)”的功能, 将分布式的处理结果汇总统计</li>
</ul>
</li>
</ul>
<p>用户可以使用MapReduce框架完成自定义需求的程序开发</p>
<p>MapReduce的功能接口可以使用Java, Python等语言进行实现</p>
<p><strong>MapReduce的运行机制</strong></p>
<ul>
<li>将执行的需求分解为多个Map Task和Reduce Task</li>
<li>将Map Task和Reduce Task分配到对应的服务器去执行</li>
</ul>
<h3 id="YARN的概述"><a href="#YARN的概述" class="headerlink" title="YARN的概述"></a>YARN的概述</h3><blockquote>
<p>  YARN是Hadoop内提供的进行分布式资源调度的组件</p>
</blockquote>
<h4 id="资源调度"><a href="#资源调度" class="headerlink" title="资源调度"></a>资源调度</h4><ul>
<li>资源: 服务器硬件资源, 如: CPU, 内存, 硬盘, 网络</li>
<li>资源调度: 管控服务器硬件资源, 提供更高的利用率</li>
<li>分布式资源调度: 管控整个分布式服务器集群的全部资源, 整合进行统一调度</li>
</ul>
<p><font color=cpink>对于资源的利用, 有规划, 有管理的调度资源使用, 是最高效的方式</font></p>
<h4 id="YARN的资源调度"><a href="#YARN的资源调度" class="headerlink" title="YARN的资源调度"></a>YARN的资源调度</h4><p>YARN管控整个集群的资源进行调度, 应用程序在运行时, 就是在YARN的管理下运行的</p>
<ul>
<li>程序向YARN申请所需资源</li>
<li>YARN为程序分配所需资源供程序使用</li>
</ul>
<h4 id="YARN和MapReduce的关系"><a href="#YARN和MapReduce的关系" class="headerlink" title="YARN和MapReduce的关系"></a>YARN和MapReduce的关系</h4><ul>
<li>YARN用来调度资源给MapReduce分配和管理运行资源</li>
<li>MapReduce需要YARN才能执行(普遍情况下)</li>
</ul>
<h4 id="YARN架构-M-S架构"><a href="#YARN架构-M-S架构" class="headerlink" title="YARN架构(M-S架构)"></a>YARN架构(M-S架构)</h4><p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230415120120.png" alt="image-20230415120119059"></p>
<ul>
<li>ResourceManager: 整个集群的资源调度者, 负责调度各个程序所需的资源</li>
<li>NodeManager: 单个服务器的资源调度者, 负责调度单个服务器上的资源提供给应用程序使用</li>
</ul>
<p><strong>YARN容器</strong></p>
<ul>
<li>NodeManager预先占用这一部分资源</li>
<li>将者一部分资源提供给程序使用</li>
<li>程序运行在容器(集装箱)内, <font color='cpink'>无法突破容器的资源限制</font></li>
</ul>
<h4 id="YARN辅助角色"><a href="#YARN辅助角色" class="headerlink" title="YARN辅助角色"></a>YARN辅助角色</h4><p>辅助角色使得YARN集群运行更加稳定</p>
<ul>
<li>代理服务器(ProxyServer): Web Application Proxy Web应用程序代理</li>
<li>历史服务器(JobHistoryServer): 应用程序历史信息记录服务</li>
</ul>
<h5 id="代理服务器-ProxyServer"><a href="#代理服务器-ProxyServer" class="headerlink" title="代理服务器(ProxyServer)"></a>代理服务器(ProxyServer)</h5><p>代理服务器, 即Web应用代理是YARN的一部分, 默认情况下, 它将作为资源管理器(RM)的一部分运行, 也可以更改配置为独立模式下运行. 使用代理可以减少通过YARN进行基于网络的攻击的可能性.</p>
<blockquote>
<p>  保证WEB UI访问的安全性</p>
</blockquote>
<p>分离代理服务器配置<code>yarn-site.xml</code>:</p>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.web-proxy.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>node1:8089<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">description</span>&gt;</span>代理服务器主机和端口<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<p>并通过命令启动:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_">$</span><span class="language-bash">HADOOP_YARN_HOME/sbin/yarn-daemon.sh start proxyserver</span></span><br></pre></td></tr></table></figure>

<h5 id="历史服务器-JobHistoryServer"><a href="#历史服务器-JobHistoryServer" class="headerlink" title="历史服务器(JobHistoryServer)"></a>历史服务器(JobHistoryServer)</h5><p>记录历史运行的程序的信息以及产生的日志并提供WEB UI站点供用户浏览器查看</p>
<blockquote>
<p>  记录历史程序运行信息和日志</p>
</blockquote>
<p>配置:</p>
<ul>
<li>开启日志聚合, 从容器中抓取日志到HDFS集中存储</li>
</ul>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.log.aggregation-enable<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>true<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">description</span>&gt;</span>开启日志聚合<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.remote-app-log-dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>/tmp/logs<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">description</span>&gt;</span>程序日志HDFS的存储路径<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<ul>
<li>配置历史服务器端口和主机</li>
</ul>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.jobhistory.webapp.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">value</span>&gt;</span>node1:19888<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">description</span>&gt;</span>历史服务器web端口为node1的19888<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<h4 id="YARN部署"><a href="#YARN部署" class="headerlink" title="YARN部署"></a>YARN部署</h4><ol>
<li>MapRecude配置文件</li>
</ol>
<p>在<code>$HADOOP_HOME/etc/hadoop</code> 文件夹中</p>
<ul>
<li><code>mapred-env.sh</code></li>
</ul>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 设置JDK路径</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/export/server/jdk</span><br><span class="line"><span class="comment"># 设置JobHistoryServer进程内存为1G</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_JOB_HISTORYSERVER_HEAPSIZE=1000</span><br><span class="line"><span class="comment"># 设置日志级别为INFO</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_MAPRED_ROOT_LOGGER=INFO,RFA</span><br></pre></td></tr></table></figure>

<ul>
<li><code>mapred-site.xml</code></li>
</ul>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.framework.name<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>yarn<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">description</span>&gt;</span>MapReduce的运行框架设置为YARN<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.jobhistory.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>node1:10020<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">descrpition</span>&gt;</span>历史服务器通讯端口为node1:10020<span class="tag">&lt;/<span class="name">descrpition</span>&gt;</span>&gt;</span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.jobhistory.webapp.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>node1:19888<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">descrpition</span>&gt;</span>历史服务器web端口为node1:19888<span class="tag">&lt;/<span class="name">descrpition</span>&gt;</span>&gt;</span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.jobhistory.intermediate-done-dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>/data/mr-history/tmp<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">descrpition</span>&gt;</span>历史信息在HDFS的记录临时路径<span class="tag">&lt;/<span class="name">descrpition</span>&gt;</span>&gt;</span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.jobhistory.done-dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>/data/mr-history/done<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">descrpition</span>&gt;</span>历史信息在HDFS的记录路径<span class="tag">&lt;/<span class="name">descrpition</span>&gt;</span>&gt;</span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.jobhistory.am.env<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>HADOOP_MAPRED_HOME=$HADOOP_HOME<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">descrpition</span>&gt;</span>MapReduce Home 设置为HADOOP_HOME<span class="tag">&lt;/<span class="name">descrpition</span>&gt;</span>&gt;</span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.map.env<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>HADOOP_MAPRED_HOME=$HADOOP_HOME<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">descrpition</span>&gt;</span>MapReduce Home 设置为HADOOP_HOME<span class="tag">&lt;/<span class="name">descrpition</span>&gt;</span>&gt;</span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">	<span class="tag">&lt;<span class="name">name</span>&gt;</span>mapreduce.reduce.env<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>HADOOP_MAPRED_HOME=$HADOOP_HOME<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">descrpition</span>&gt;</span>MapReduce Home 设置为HADOOP_HOME<span class="tag">&lt;/<span class="name">descrpition</span>&gt;</span>&gt;</span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<ol start="2">
<li>YRAN配置文件</li>
</ol>
<p>在<code>$HADOOP_HOME/etc/hadoop</code>文件夹内</p>
<ul>
<li><code>yarn-env.sh</code></li>
</ul>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment"># 设置JDK路径</span></span><br><span class="line"><span class="built_in">export</span> JAVA_HOME=/export/server/jdk</span><br><span class="line"><span class="comment"># 设置HADOOP_HOME的环境变量</span></span><br><span class="line"><span class="built_in">export</span> HDAOOP_HOME=/export/server/hadoop</span><br><span class="line"><span class="comment"># 设置配置文件路径的环境变量</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_CONF_DIR=<span class="variable">$HADOOP_HOME</span>/etc/hadoop</span><br><span class="line"><span class="comment"># 设置日志文件路径的环境变量</span></span><br><span class="line"><span class="built_in">export</span> HADOOP_LOG_DIR=<span class="variable">$HADOOP_HOME</span>/logs</span><br></pre></td></tr></table></figure>

<ul>
<li><code>yarn-site.xml</code></li>
</ul>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.resourcemanager.hostname<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">value</span>&gt;</span>node1<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">description</span>&gt;</span>ResourceManager设置在node1节点<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.local-dirs<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">value</span>&gt;</span>/data/nm-local<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">description</span>&gt;</span>NodeManager中间数据本地存储路径<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.log-dirs<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">value</span>&gt;</span>/data/nm-log<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">description</span>&gt;</span>NodeManager数据日志本地储存路径<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.aux-services<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">value</span>&gt;</span>mapreduce_shuffle<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">description</span>&gt;</span>为MapReduce程序开启Shuffle服务<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.log.server.url<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">value</span>&gt;</span>http://node1:19888/jobhistory/logs<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">description</span>&gt;</span>历史服务器URL<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.web-proxy.address<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">value</span>&gt;</span>node1:8089<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">description</span>&gt;</span>代理服务器主机和端口<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.log-aggregation-enable<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">value</span>&gt;</span>true<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">description</span>&gt;</span>开启日志聚合<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.nodemanager.remote-app-log-dir<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">value</span>&gt;</span>/tmp/logs<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">description</span>&gt;</span>程序日志HDFS的尺寸路径<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">name</span>&gt;</span>yarn.resourcemanager.scheduler.class<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">value</span>&gt;</span>org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">description</span>&gt;</span>选择公平调度器<span class="tag">&lt;/<span class="name">description</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br></pre></td></tr></table></figure>

<ol start="3">
<li>分发配置文件</li>
</ol>
<p>MapReduce和YARN配置好后, 需要分发到其他的服务器节点中</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">scp mapred-env.sh mapred-site.xml yarn-evn.sh yarn-site.xml node2:`pwd`/</span><br><span class="line">scp mapred-env.sh mapred-site.xml yarn-evn.sh yarn-site.xml node3:`pwd`/</span><br></pre></td></tr></table></figure>

<ol start="4">
<li>集群启动命令</li>
</ol>
<ul>
<li>一键启动&#x2F;停止</li>
</ul>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">start-yarn.sh</span><br><span class="line">stop-yarn.sh</span><br></pre></td></tr></table></figure>

<ul>
<li>单独启动&#x2F;停止</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">yarn --daemon start|stop resourcemanager|nodemanager|proxyserver</span><br></pre></td></tr></table></figure>

<ul>
<li>历史服务器启动&#x2F;停止</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">mapred --daemon start|stop historyserver</span><br></pre></td></tr></table></figure>

<ol start="5">
<li>在浏览器中查看</li>
</ol>
<p><a target="_blank" rel="noopener" href="http://node1:8088/">http://node1:8088</a></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230415131916.png" alt="image-20230415131913576"></p>
<h4 id="体验-YARN"><a href="#体验-YARN" class="headerlink" title="体验: YARN"></a>体验: YARN</h4><p>YARN作为资源调度管控框架, 其本身提供资源供许多程序运行, 常见的有:</p>
<ul>
<li>MapReduce程序</li>
<li>Spark程序</li>
<li>Flink程序</li>
</ul>
<h5 id="提交MapReduce程序至YARN运行"><a href="#提交MapReduce程序至YARN运行" class="headerlink" title="提交MapReduce程序至YARN运行"></a>提交MapReduce程序至YARN运行</h5><p>Hadoop官方内置了一些预置的MapReduce程序代码, 可以通过命令直接调用</p>
<ul>
<li>wordcount: 单词计数程序</li>
</ul>
<p>统计指定文件内各个单词出现的次数</p>
<ul>
<li>pi: 求圆周率</li>
</ul>
<p>通过蒙特卡洛算法(统计模拟法)求圆周率</p>
<blockquote>
<p>  这些内置的示例程序都在$HADOOP_HOME&#x2F;share&#x2F;hadoop&#x2F;mapreduce&#x2F;hadoop-mapreduce-examples-3.3.5.jar中</p>
<p>  可以通过<code>hadoop jar</code>命令运行, 提交MapRedue程序到YARN中</p>
</blockquote>
<p>语法: <code>hadoop jar 程序文件 java类名 [程序参数]</code></p>
<h6 id="提交wordcount示例程序"><a href="#提交wordcount示例程序" class="headerlink" title="提交wordcount示例程序"></a>提交wordcount示例程序</h6><p>单词计数示例程序的功能很简单</p>
<ul>
<li>给定数据输入的路径(HDFS), 给定结果输出的路径(HDFS)</li>
<li>将输入路径内的数据中的单词进行计数, 将结果写到输出路径</li>
</ul>
<p>过程如下:</p>
<ol>
<li>向YARN提交程序</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hadoop jar /export/server//hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.5.jar wordcount hdfs://node1:8020/input hdfs://node1:8020/output/wc</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230415150810.png" alt="image-20230415150807963"></p>
<ol start="2">
<li>查看生成文件</li>
</ol>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230415150854.png" alt="image-20230415150850741"></p>
<p>其中<code>_SUCCESS</code>没有内容, 表明状态(成功)</p>
<ol start="3">
<li>查看文件内容</li>
</ol>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230415150954.png" alt="image-20230415150952946"></p>
<p>给定文件内容如下:</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230415151041.png" alt="image-20230415151039830"></p>
<p>并且在浏览器中可以查看过程</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230415151329.png" alt="image-20230415151326972"></p>
<h6 id="提交求圆周率示例程序"><a href="#提交求圆周率示例程序" class="headerlink" title="提交求圆周率示例程序"></a>提交求圆周率示例程序</h6><p>使用蒙特卡洛算法模拟计算圆周率</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">hadoop jar /export/server//hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.3.5.jar pi 3 10000</span><br></pre></td></tr></table></figure>

<ul>
<li><code>pi</code> 表示是运行的java类, 这里是运行jar包中的求pi程序</li>
<li>参数<code>3</code> 表示设置几个map任务</li>
<li>参数<code>10000</code> 表示模拟求pi的样本数, (越大越精准, 速度会变慢)</li>
</ul>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230415151917.png" alt="image-20230415151915903"></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230415152208.png" alt="image-20230415152206744"></p>
<h2 id="Apache-Hive-入门"><a href="#Apache-Hive-入门" class="headerlink" title="Apache Hive(入门)"></a>Apache Hive(入门)</h2><p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230415153303.png" alt="image-20230415153301067"></p>
<blockquote>
<p>  分布式SQL计算平台</p>
</blockquote>
<p>大数据体系中充斥着非常多的统计分析场景, 所以使用SQL去处理数据, 在大数据中也有极大的需求</p>
<p>Apache Hive的主要功能:</p>
<ul>
<li>将<font color=cpink>SQL语句翻译成MapReduce程序</font>运行</li>
</ul>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230415153748.png" alt="image-20230415153746713"></p>
<p>基于Hive为用户提供了分布式SQL计算的能力, 写的是SQL, 执行的是MapReduce</p>
<p>使用Hive处理数据的优点:</p>
<ul>
<li>操作接口采用<font color=cyel>类SQL语句</font>, 提供快速开发能力(简单, 容易上手)</li>
<li>底层执行MapReduce, <font color=cyel>可以完成分布式海量数据的SQL处理</font></li>
</ul>
<h3 id="Hive架构"><a href="#Hive架构" class="headerlink" title="Hive架构"></a>Hive架构</h3><p>Apache Hive 的2大主要组件就是: SQL解析器和元数据存储</p>
<ul>
<li>元数据储存</li>
</ul>
<p>通常是储存在关系型数据库如<code>mysql|derby</code>中, Hive中的元数据包括表的名字, 表的列和分区及其属性, 表的属性, 表的数据所在目录</p>
<p>– Hive提供了 <font color=coink>Metastore </font> 服务进程提供元数据管理功能</p>
<ul>
<li>(SQL解析器)Driver驱动程序, 包括语法解析器, 计划编译器, 优化器, 执行器</li>
</ul>
<p>完成HQL查询语句从词法分析, 语法分析, 编译, 优化以及查询计划的生成. 生成的查询计划存储在HDFS中, 随后有执行引擎调用执行</p>
<ul>
<li>用户接口</li>
</ul>
<p>包括CLI, JDBC&#x2F;ODBC, WebGUI, 其中CLI为(command line interface)命令行; Hive中的Thrift服务器运行外部客户端通过网络与Hive进行交互, 类似于JDBC或ODBC, WebGUI通常是浏览器访问方式</p>
<h3 id="VMware部署Hive"><a href="#VMware部署Hive" class="headerlink" title="VMware部署Hive"></a>VMware部署Hive</h3><p>Hive是单机工具, 只需要部署在一台服务器上即可</p>
<p>Hive虽然是单机的, 但是它可以提交分布式运行的MapReduce程序运行</p>
<ol>
<li>安装Mariadb数据库(node1), 并配置权限</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">yum安装Mariadb</span></span><br><span class="line">yum install mariadb-server</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">启动MySQL开机自启动</span></span><br><span class="line">systemctl start mariadb</span><br><span class="line">systemctl enable mariadb</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">检查MySQL服务状态</span></span><br><span class="line">systemctl status mariadb</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">MariaDB初始化配置</span></span><br><span class="line">mysql_secure_installation</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">第一个默认回车</span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">第二个设置密码选择y, 输入root(再次输入root)</span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">之后的配置全选y</span></span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">登陆数据库检查</span></span><br><span class="line">mysql -uroot -p</span><br><span class="line">root</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">成功登陆则没有问题</span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">为root用户配置权限(后面会用到)</span></span><br><span class="line">GRANT ALL PRIVILEGES ON *.* TO &#x27;root&#x27;@&#x27;%&#x27; IDENTIFIED BY &#x27;root&#x27; WITH GRANT OPTION; </span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">刷新权限, 使权限生效</span></span><br><span class="line">FLUSH PRIVILEGES; </span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">退出数据库</span></span><br><span class="line">exit;</span><br></pre></td></tr></table></figure>

<ol start="2">
<li>上传hive并解压</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">tar -zxvf /export/apache-hive-3.1.3-bin.tar.gz /export/server</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">设置软链接</span></span><br><span class="line">ln -s /export/server/apache-hive-3.1.3-bin /export/server/hive</span><br></pre></td></tr></table></figure>

<ol start="3">
<li>配置hive</li>
</ol>
<ul>
<li>在Hive的conf目录中, 新建<code>hive-env.sh</code>文件</li>
</ul>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">export</span> HADOOP_HOME=/export/server/hadoop</span><br><span class="line"><span class="built_in">export</span> HIVE_CONF_DIR=/export/server/hive</span><br><span class="line"><span class="built_in">export</span> HIVE_AUX_JARS_PATH=/export/server/hive/lib</span><br></pre></td></tr></table></figure>

<ul>
<li>在Hive的conf目录中, 新建<code>hive-site.xml</code>文件</li>
</ul>
<figure class="highlight xml"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br></pre></td><td class="code"><pre><span class="line"><span class="tag">&lt;<span class="name">configuration</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>javax.jdo.option.ConnectionURL<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>jdbc:mysql://node1:3306/hive?createDatabaseIfNotExist=true<span class="symbol">&amp;amp;</span>useSSL=false<span class="symbol">&amp;amp;</span>useUnicode=true<span class="symbol">&amp;amp;</span>characterEncoding=UTF-8<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">  <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>javax.jdo.option.ConnectionDriverName<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>com.mysql.jdbc.Driver<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">  <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>javax.jdo.option.ConnectionUserName<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>root<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">  <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>javax.jdo.option.ConnectionPassword<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>root<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">  <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.server2.thrift.bind.host<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>node1<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">  <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.metastore.uris<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>thrift://node1:9083<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line">  <span class="tag">&lt;<span class="name">property</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">name</span>&gt;</span>hive.metastore.event.db.notification.api.auth<span class="tag">&lt;/<span class="name">name</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">value</span>&gt;</span>false<span class="tag">&lt;/<span class="name">value</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;/<span class="name">property</span>&gt;</span></span><br><span class="line"></span><br><span class="line"><span class="tag">&lt;/<span class="name">configuration</span>&gt;</span></span><br></pre></td></tr></table></figure>

<ol start="4">
<li>上传<code>mysql-connector-java-5.1.34.jar</code>到Hive的lib目录中</li>
</ol>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">mv /export/mysql-connector-java-5.1.34.jar /export/server/hive/lib</span><br></pre></td></tr></table></figure>

<ol start="5">
<li>初始化元数据库</li>
</ol>
<ul>
<li>在Mariadb中新建数据库:hive</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">CREATE DATABASE hive CHARSET UTF8;</span><br></pre></td></tr></table></figure>

<ul>
<li>执行元数据库初始化命令</li>
</ul>
<figure class="highlight sh"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="built_in">cd</span> /export/server/hive</span><br><span class="line">bin/schematool -initSchema -dbType mysql -verbos --verbose</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416082612.png" alt="image-20230416082611766"></p>
<p>显示<code>schemaTool completed</code>则成功</p>
<ol start="6">
<li>启动hive(使用hadoop用户)</li>
</ol>
<ul>
<li>将hive文件所属为hadoop用户</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">在root登陆中将文件所属该为hadoop用户</span></span><br><span class="line">chown -R hadoop:hadoop apache-hive-3.1.3-bin hive</span><br></pre></td></tr></table></figure>

<ul>
<li>创建一个logs文件夹</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">mkdir /export/server/hive/logs</span><br></pre></td></tr></table></figure>

<ul>
<li>启动元数据管理服务(必须启动, 否则无法正常工作)</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">前台启动</span></span><br><span class="line">bin/hive --service metastore</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">后台启动</span></span><br><span class="line">nohup bin/hive --service metastore &gt;&gt; logs/metastore.log 2&gt;&amp;1 &amp;</span><br></pre></td></tr></table></figure>

<ul>
<li>启动客户端(二选一)</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">先启动Hadoop</span></span><br><span class="line">start-dfs.sh</span><br><span class="line">start-yarn.sh</span><br><span class="line">mapred --daemon start historyserver</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">当前选择这种方式 (可以直接写SQL语句)</span></span><br><span class="line">Hive Sell: bin/hive</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">不可直接写SQL, 需要外部客户端链接使用</span></span><br><span class="line">Hive ThriftServer: bin/hive --service hiveserver2 </span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416091124.png" alt="image-20230416091121522"></p>
<p>表明启动成功</p>
<blockquote>
<p>  如果出现报错: Cannot create directory &#x2F;tmp&#x2F;hive. Name node is in safe mod</p>
<p>  可以输入以下命令进行解决</p>
</blockquote>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">退出安全模式</span></span><br><span class="line">hdfs dfsadmin -safemode leave</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">强制退出安全模式(第一个不管用时再选择)</span></span><br><span class="line">hdfs dfsadmin -safemode forceExit</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416091932.png" alt="image-20230416091931196"></p>
<p>到这里表明Hive已经部署成功了</p>
<h3 id="Hive体验"><a href="#Hive体验" class="headerlink" title="Hive体验"></a>Hive体验</h3><p>确保已经启动了Metastore服务</p>
<ul>
<li>创建表</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">create table test(id int, name string, gender string);</span><br></pre></td></tr></table></figure>

<ul>
<li>插入数据</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">insert into test values(1, &#x27;李白&#x27;, &#x27;男&#x27;), (2, &#x27;张凯&#x27;, &#x27;男&#x27;), (3, &#x27;无患子&#x27;,&#x27;女&#x27;);</span><br></pre></td></tr></table></figure>

<ul>
<li>查询数据</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line"># 简单查询, 不走MapRuduce</span><br><span class="line">select * from test;</span><br><span class="line"># 复杂查询, 调用MapReduce</span><br><span class="line">select gender, COUNT(*) as cnt from test group by gender;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416103056.png" alt="image-20230416103055851"></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416103127.png" alt="image-20230416103126333"></p>
<ul>
<li>数据存储路径</li>
</ul>
<blockquote>
<p>  数据表存储在HDFS的&#x2F;user&#x2F;hive&#x2F;warehouse中</p>
</blockquote>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416103534.png" alt="image-20230416103531396"></p>
<h3 id="HiveServer2-amp-Beeline"><a href="#HiveServer2-amp-Beeline" class="headerlink" title="HiveServer2 &amp; Beeline"></a>HiveServer2 &amp; Beeline</h3><p>在启动Hive的时候, 除了必备的Metastore服务外, 有两种启动Hive的方式</p>
<ul>
<li><p><code>bin/hive</code> Hive的Shell客户端, 可以直接写SQL</p>
</li>
<li><p><code>bin/hive --service hiveserver2</code> </p>
<ul>
<li>后台执行脚本<code>nohup bin/hive --service hiverserver2 &gt;&gt; logs/hiveserver2.log 2&gt;&amp;1 &amp;</code></li>
</ul>
</li>
</ul>
<blockquote>
<p>  bin&#x2F;hive –service metastore 启动元数据管理服务</p>
<p>  bin&#x2F;hive –service hiveserver 启动HiveServer2服务</p>
</blockquote>
<p>HiveServer2是Hive内置的ThriftServer服务, 提供Thrift端口供其他客户端链接, 可以连接ThriftServer的客户端有:</p>
<ul>
<li>Hive内置的beeline客户端(命令行工具)</li>
<li>第三方图形化SQL工具, 如DataGrip, PyCharm等</li>
</ul>
<h4 id="启动HiveServer2"><a href="#启动HiveServer2" class="headerlink" title="启动HiveServer2"></a>启动HiveServer2</h4><p>在安装hive的服务器上, 首先启动<code>metastore</code>服务, 再启动<code>hiveserver2</code>服务</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">nohup bin/hive --service metastore &gt;&gt; logs/metastore.log 2&gt;&amp;1 &amp;</span><br><span class="line">nohup bin/hive --service hiveserver2 &gt;&gt; logs/hiveserver2.log 2&gt;&amp;1 &amp;</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">设置本地模式</span></span><br><span class="line">set hive.exec.mode.local.auto=true;</span><br></pre></td></tr></table></figure>

<p>默认Thrift端口号为: 10000</p>
<h4 id="beeline"><a href="#beeline" class="headerlink" title="beeline"></a>beeline</h4><ul>
<li>在node1上使用的beeline客户端进行连接访问</li>
<li>beeline是jdbc的客户端, 通过jdbc协议和HiveServer2服务进行通信, 通信地址是<code>jdbc:hive2://node1:10000</code></li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">启动beeline</span></span><br><span class="line">bin/beeline</span><br><span class="line"><span class="meta prompt_"></span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">链接服务</span></span><br><span class="line">!connect jdbc:hive2://node1:10000</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">输入用户名: hadoop</span></span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">输入密码: hadoop(我的密码是hadoop)</span></span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416110257.png" alt="image-20230416110118099"></p>
<blockquote>
<p>  输入 show databaes; show tables; 均可以正常显示</p>
</blockquote>
<h3 id="HiveServer-amp-PyCharm"><a href="#HiveServer-amp-PyCharm" class="headerlink" title="HiveServer &amp; PyCharm"></a>HiveServer &amp; PyCharm</h3><p>node1启动HiveServer2后, 可以在PyCharm中添加数据源</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416121458.png" alt="image-20230416121455767"></p>
<p>下载驱动后, 测试连接成功则可以使用(此处用户名为: hadoop, 密码为: hadoop, 与上同)</p>
<p><img src="C:\Users\Crappy\AppData\Roaming\Typora\typora-user-images\image-20230416121742088.png" alt="image-20230416121742088"></p>
<h2 id="Apache-Hive-使用"><a href="#Apache-Hive-使用" class="headerlink" title="Apache Hive(使用)"></a>Apache Hive(使用)</h2><h3 id="数据库操作语法"><a href="#数据库操作语法" class="headerlink" title="数据库操作语法"></a>数据库操作语法</h3><h4 id="数据库操作"><a href="#数据库操作" class="headerlink" title="数据库操作"></a>数据库操作</h4><ul>
<li>创建数据库</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">create database if not existes myhive;</span><br><span class="line"></span><br><span class="line">use myhive;</span><br></pre></td></tr></table></figure>

<ul>
<li>查看数据库详细信息</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">desc database myhive;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416122339.png" alt="image-20230416122337865"></p>
<blockquote>
<p>  hive的数据库在hdfs上本质上就是一个文件夹</p>
<p>  默认数据库的存放路径是HDFS的: <code>/user/hive/warehouse</code></p>
</blockquote>
<ul>
<li>创建数据库并指定hdfs的存储位置</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">create database myhive2 location &#x27;user/hive/myhive2&#x27;;</span><br><span class="line">desc database myhive2;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416122853.png" alt="image-20230416122852202"></p>
<ul>
<li>删除数据库, 如果数据库中有数据表, 则会报错</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">drop database myhive;</span><br></pre></td></tr></table></figure>

<ul>
<li>强制删除数据库, 包括数据库中的数据表</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">use myhive2;</span><br><span class="line">-- 创建一个表</span><br><span class="line">create table test(id int, name string);</span><br><span class="line">-- 之际删除会报错</span><br><span class="line">drop database myhive2;</span><br><span class="line">-- 强制删除, 则会删除数据库, 及其所有的数据表</span><br><span class="line">drop database myhive2 cascade;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416123427.png" alt="image-20230416123425970"></p>
<h3 id="数据表操作语法"><a href="#数据表操作语法" class="headerlink" title="数据表操作语法"></a>数据表操作语法</h3><h3 id="创建表的语法"><a href="#创建表的语法" class="headerlink" title="创建表的语法"></a>创建表的语法</h3><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">create [external] table [if not exists] table_name</span><br><span class="line">	[(col_name data_type [comment col_comment], ...)]</span><br><span class="line">	[comment talbe_comment]</span><br><span class="line">	[partitioned by (col_name data_type [comment col_comment], ...)]</span><br><span class="line">	[clustered by (col_name, col_name, ...)]</span><br><span class="line">	[sorted by (col_name [asc|desc], ...) into num_buckets buckets]</span><br><span class="line">	[row format row_format]</span><br><span class="line">	[stored as file_format]</span><br><span class="line">	[location hdfs_path]</span><br></pre></td></tr></table></figure>

<ul>
<li><code>external</code> 创建外部表</li>
<li><code>partitioned by</code> 分区表</li>
<li><code>clustered by</code> 分桶表</li>
<li><code>stored as</code> 存储格式</li>
<li><code>location</code> 存储位置</li>
</ul>
<p>hive所支持的数据类型</p>
<table>
<thead>
<tr>
<th align="center">分类</th>
<th align="center">类型</th>
<th>描述</th>
<th align="center">字面量实例</th>
</tr>
</thead>
<tbody><tr>
<td align="center">原始类型</td>
<td align="center">boolean</td>
<td>true&#x2F;false</td>
<td align="center">TRUE</td>
</tr>
<tr>
<td align="center"></td>
<td align="center">tinyint</td>
<td>1字节的有符号整数: -127-128</td>
<td align="center">1Y</td>
</tr>
<tr>
<td align="center"></td>
<td align="center">smallint</td>
<td>2字节的有符号整数: -32768-32767</td>
<td align="center">1S</td>
</tr>
<tr>
<td align="center"></td>
<td align="center"><font color=cpink>int</font></td>
<td>4字节的有符号整数</td>
<td align="center">1</td>
</tr>
<tr>
<td align="center"></td>
<td align="center">bigint</td>
<td>8字节的有符号整数</td>
<td align="center">1L</td>
</tr>
<tr>
<td align="center"></td>
<td align="center">float</td>
<td>4字节单精度浮点数</td>
<td align="center">1.0</td>
</tr>
<tr>
<td align="center"></td>
<td align="center"><font color=cpink>double</font></td>
<td>8字节双精度浮点数</td>
<td align="center">1.0</td>
</tr>
<tr>
<td align="center"></td>
<td align="center">deicimal</td>
<td>任意精度的带符号小数</td>
<td align="center">1.0</td>
</tr>
<tr>
<td align="center"></td>
<td align="center"><font color=cpink>string</font></td>
<td>字符串, 可变长</td>
<td align="center"></td>
</tr>
<tr>
<td align="center"></td>
<td align="center"><font color=cpink>varchar</font></td>
<td>变长字符串</td>
<td align="center"></td>
</tr>
<tr>
<td align="center"></td>
<td align="center">char</td>
<td>固定长字符串</td>
<td align="center"></td>
</tr>
<tr>
<td align="center"></td>
<td align="center">binary</td>
<td>字节数组</td>
<td align="center"></td>
</tr>
<tr>
<td align="center"></td>
<td align="center"><font color=cpink>timestamp</font></td>
<td>时间戳, 毫秒精确度</td>
<td align="center">122637267291</td>
</tr>
<tr>
<td align="center"></td>
<td align="center"><font color=cpink>date</font></td>
<td>日期</td>
<td align="center">‘2023-04-16’</td>
</tr>
<tr>
<td align="center">复杂类型</td>
<td align="center">array</td>
<td>有序的同类数据集合</td>
<td align="center">array(1, 2)</td>
</tr>
<tr>
<td align="center"></td>
<td align="center">map</td>
<td>key-val, key必须为原始2类型, val可以是任意类型</td>
<td align="center">map(‘a’, 1, ‘b’, 2)</td>
</tr>
<tr>
<td align="center"></td>
<td align="center">struct</td>
<td>字段集合, 类型可以不同</td>
<td align="center"></td>
</tr>
<tr>
<td align="center"></td>
<td align="center">union</td>
<td>在有限取值范围内的一个值</td>
<td align="center"></td>
</tr>
</tbody></table>
<ul>
<li>基础表创建</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">-- 在myhive中创建</span><br><span class="line">create table my_test(</span><br><span class="line">	id int,</span><br><span class="line">    name string,</span><br><span class="line">    gender string</span><br><span class="line">);</span><br><span class="line"></span><br><span class="line">-- 如果在其他数据库中, 想要对myhive创建表</span><br><span class="line">create table myhive.my_test_2(</span><br><span class="line">	id int,</span><br><span class="line">);</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416130334.png"></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230416130659.png"></p>
<ul>
<li>删除表</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">drop table tab</span><br></pre></td></tr></table></figure>

<h3 id="Hive的表类型"><a href="#Hive的表类型" class="headerlink" title="Hive的表类型"></a>Hive的表类型</h3><p>在Hive中可以创建的表的类型有4种, 分别是:</p>
<ol>
<li>内部表</li>
<li>外部表</li>
<li>分区表</li>
<li>分桶表</li>
</ol>
<ul>
<li>内部表</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">create table table_name ...</span><br></pre></td></tr></table></figure>

<p>未被<code>external</code>关键字修饰的是内部表, 即普通表. 内部表又称管理表, 内部表数据存储的位置由<code>hive.metastore.warehouse.dir</code>参数决定(默认是:<code>/user/hive/warehouse</code>), 删除内部表会直接<font color='cpink'>删除元数据(metadata) 及存储数据</font>, 因此内部表不适合和其他工具共享数据</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre></td><td class="code"><pre><span class="line">-- 创建学生表(内部表)</span><br><span class="line">create database if not exists myhive;</span><br><span class="line">use myhive;</span><br><span class="line">create table if not exists stu(</span><br><span class="line">    id int,</span><br><span class="line">    name string</span><br><span class="line">);</span><br><span class="line">insert into stu values (1, &#x27;李白&#x27;), (2, &#x27;张飞&#x27;);</span><br><span class="line">select * from stu;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417191248.png" alt="image-20230417191247697"></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417191515.png" alt="image-20230417191513449"></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417191605.png" alt="image-20230417191603657"></p>
<ul>
<li>数据分隔符</li>
</ul>
<p>如上图的<code>/myhive.db/stu/000000_0</code>, 默认的数据分隔符是:<code>\001</code>, 明文不可见</p>
<ul>
<li>自行定义分隔符</li>
</ul>
<p>在创建表的时候可以自己设定</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">create table if not exists stu2(</span><br><span class="line">	id int,</span><br><span class="line">    name string</span><br><span class="line">) row format delimited fields terminated by &#x27;\t&#x27;;</span><br><span class="line">-- 表示以 \t 为分隔符</span><br><span class="line">insert into stu_2 values(1, &#x27;Lili&#x27;), (2, &#x27;mark&#x27;);</span><br><span class="line">select * from stu_2;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417193450.png" alt="image-20230417193449305"></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417193544.png" alt="image-20230417193543761"></p>
<ul>
<li>外部表</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">create external table table_name ... location ...</span><br></pre></td></tr></table></figure>

<p>被<code>external</code>关键字修饰的是外部表, 即管理表.</p>
<p>外部表是指表数据可以在任何位置, 通过<code>location</code>关键字指定. 数据存储的不同也代表了这个表在理念上并不是在Hive内部管理的, 而是可以随意临时链接到外部数据. 所以在删除外部表的时候, <font color='cpink'>仅仅是删除元数据(表的信息), 不会删除数据本身</font>.</p>
<blockquote>
<p>  内部表和外部表对比</p>
</blockquote>
<table>
<thead>
<tr>
<th align="center"></th>
<th align="center">创建</th>
<th align="center">存储位置</th>
<th align="center">删除数据</th>
<th align="center">理念</th>
</tr>
</thead>
<tbody><tr>
<td align="center">内部表</td>
<td align="center">create table</td>
<td align="center">hive管理, <br />默认&#x2F;user&#x2F;hive&#x2F;warehouse</td>
<td align="center">· 删除元数据(表信息)<br />· 删除数据</td>
<td align="center">Hive管理表<br />持久使用</td>
</tr>
<tr>
<td align="center">外部表</td>
<td align="center">create external table</td>
<td align="center">随意, location关键字指定</td>
<td align="center">· 仅删除元数据(表信息)<br />· 保留数据</td>
<td align="center">临时链接<br />外部数据使用</td>
</tr>
</tbody></table>
<ol>
<li>先有表, 后有数据</li>
</ol>
<p>先创建外部表, 再将数据文件移动到指定的<code>location</code>目录中</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">create external table test_ex1(</span><br><span class="line">    id int,</span><br><span class="line">    name string</span><br><span class="line">) row format delimited fields terminated by &#x27;\t&#x27; location &#x27;/tmp/test_ex1&#x27;;</span><br></pre></td></tr></table></figure>

<p>准备一个数据文件<code>data.txt</code>, 以<code>\t</code>分隔</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417195539.png" alt="image-20230417195535063"></p>
<p>将数据文件上传到hdfs中的<code>/tmp/test_ex1</code>中</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417195642.png" alt="image-20230417195641185"></p>
<p>执行<code>select</code>语句</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417195721.png" alt="image-20230417195720065"></p>
<ol start="2">
<li>先有数据, 后有表</li>
</ol>
<p>先创建一个<code>tmp/test_ex2</code>的目录, 并且将数据文件上传<code>hdfs dfs -put data.txt /tmp/test_ex2/</code>中</p>
<p>创建一个外部表, 并执行查询语句</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">create external table test_ex2(</span><br><span class="line">    id int,</span><br><span class="line">    name string</span><br><span class="line">) row format delimited fields terminated by &#x27;\t&#x27; location &#x27;/tmp/test_ex2&#x27;;</span><br><span class="line">select * from test_ex2;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417200417.png" alt="image-20230417200415053"></p>
<ul>
<li>内\外部表转换</li>
</ul>
<p>查看表类型</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">desc formatted table_name;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417201326.png" alt="image-20230417201325361"></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417201406.png" alt="image-20230417201405191"></p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">-- 内部表改外部表</span><br><span class="line">alter table table_name set tblproperties (&quot;EXTERNAL&quot;=&quot;TRUE&quot;);</span><br><span class="line">-- 外部表该内部表</span><br><span class="line">alter table table_name set tblproperties (&quot;EXTERNAL&quot;=&quot;FALSE&quot;);</span><br></pre></td></tr></table></figure>

<h3 id="数据加载和导出"><a href="#数据加载和导出" class="headerlink" title="数据加载和导出"></a>数据加载和导出</h3><h4 id="数据加载-LOAD语法"><a href="#数据加载-LOAD语法" class="headerlink" title="数据加载-LOAD语法"></a>数据加载-LOAD语法</h4><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">load data [local] inpath &#x27;linux本地 or hdfs&#x27; [overwrite] into table table_name;</span><br></pre></td></tr></table></figure>

<p>从外部将数据加载到Hive中</p>
<p>准备数据文件, 在linux本地创建<code>search.txt</code>数据文件</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">0:0:1   12345623        传感器的学习    www.123.com</span><br><span class="line">0:1:0   34528413        Python学习      www.baidu.com</span><br><span class="line">0:1:1   89411042        MySQL语法       www.sql.com</span><br><span class="line">1:0:0   55093131        用户信息        www.infoma.com</span><br></pre></td></tr></table></figure>

<p>创建数据表, 加载数据, 并查询数据</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">create table myhive.test_load(</span><br><span class="line">    dt string comment &#x27;时间&#x27;,</span><br><span class="line">    user_id string comment &#x27;用户id&#x27;,</span><br><span class="line">    search_word string comment &#x27;关键词&#x27;,</span><br><span class="line">    url string comment &#x27;网址&#x27;</span><br><span class="line">) row format delimited fields terminated by &#x27;\t&#x27;;</span><br><span class="line">-- 数据加载[本地上传 --local]</span><br><span class="line">load data local inpath &#x27;/home/hadoop/search.txt&#x27; into table myhive.test_load;</span><br><span class="line">-- 查询数据</span><br><span class="line">select * from myhive.test_load;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417205815.png" alt="image-20230417205814054"></p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">-- 数据加载[hdfs]</span><br><span class="line">load data inpath &#x27;/tmp/search.txt&#x27; into table myhive.test_load;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417210131.png" alt="image-20230417210130467"></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417210238.png" alt="image-20230417210237456"></p>
<blockquote>
<p>  <font color='cpink'>注意, 基于HDFS进行load加载数据时, 源数据文件会消失</font></p>
<p>  使用hdfs的加载, 本质上是fs -mv 的过程</p>
</blockquote>
<p><code>overwrite</code>关键字</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">load data local inpath &#x27;/home/hadoop/search.txt&#x27; overwrite into table myhive.test_load;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417210655.png" alt="image-20230417210654178"></p>
<p>使用<code>overwrite</code>关键字会对源数据进行覆盖 (默认是追加)</p>
<h4 id="数据加载-INSERT-SELECT语法"><a href="#数据加载-INSERT-SELECT语法" class="headerlink" title="数据加载-INSERT SELECT语法"></a>数据加载-INSERT SELECT语法</h4><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">insert into|overwrite table table_name select * from table;</span><br></pre></td></tr></table></figure>



<p>通过SQL语句, 从其他表中加载数据</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line">create table myhive.test_load_2(</span><br><span class="line">    dt string comment &#x27;时间&#x27;,</span><br><span class="line">    user_id string comment &#x27;用户id&#x27;,</span><br><span class="line">    search_word string comment &#x27;关键词&#x27;,</span><br><span class="line">    url string comment &#x27;网址&#x27;</span><br><span class="line">) row format delimited fields terminated by &#x27;\t&#x27;;</span><br><span class="line"></span><br><span class="line">-- 追加</span><br><span class="line">insert into myhive.test_load_2 select * from myhive.test_load;</span><br><span class="line">-- 覆盖</span><br><span class="line">insert overwrite table myhive.test_load_2 select * from myhive.test_load;</span><br></pre></td></tr></table></figure>

<h4 id="数据导出-INSERT-OVERWRITE方式"><a href="#数据导出-INSERT-OVERWRITE方式" class="headerlink" title="数据导出-INSERT OVERWRITE方式"></a>数据导出-INSERT OVERWRITE方式</h4><p>将Hive表中的数据导出到其他任意目录, 例如Linux本地磁盘, hdfs, mysql中等</p>
<p>语法</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">insert overwrite [local] directory &#x27;path&#x27; select_statement1 from from_statement;</span><br></pre></td></tr></table></figure>

<ul>
<li>导出Linux本地<ol>
<li>使用默认分隔符</li>
</ol>
</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">insert overwrite local directory &#x27;/home/hadoop/export1&#x27; select * from myhive.test_load;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417213206.png" alt="image-20230417213204681"></p>
<p>​		2. 指定分隔符</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">insert overwrite local directory &#x27;/home/hadoop/export2&#x27; row format delimited fields terminated by &#x27;\t&#x27; select * from myhive.test_load;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417213650.png" alt="image-20230417213647791"></p>
<ul>
<li>导出到HDFS上</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">insert overwrite directory &#x27;/tmp/export&#x27; row format delimited fields terminated by &#x27;\t&#x27; select * from myhive.test_load;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417213935.png" alt="image-20230417213933983"></p>
<h4 id="Hive表数据导出-hive-shell"><a href="#Hive表数据导出-hive-shell" class="headerlink" title="Hive表数据导出-hive shell"></a>Hive表数据导出-hive shell</h4><ul>
<li>基本语法</li>
</ul>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">bin/hive -e &quot;select * from myhive.test_load;&quot;</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">通过linux重定向符导入到export3.t</span></span><br><span class="line">bin/hive -e &quot;select * from myhive.test_load;&quot; &gt; /home/hadoop/export3.txt</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417214235.png" alt="image-20230417214234361"></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417214251.png" alt="image-20230417214250385"></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line"><span class="meta prompt_"># </span><span class="language-bash">创建一个export.sql的脚本文件</span></span><br><span class="line">vim export.sql</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">输入</span></span><br><span class="line">select * from myhive.test_load;</span><br><span class="line"><span class="meta prompt_"># </span><span class="language-bash">-f 执行sql脚本</span></span><br><span class="line">/export/server/hive/bin/hive -f export.sql &gt; ./export4.txt</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230417215145.png" alt="image-20230417215144826"></p>
<h3 id="分区表"><a href="#分区表" class="headerlink" title="分区表"></a>分区表</h3><p>Hive中支持多个字段作为分区, 多分区带有层级关系</p>
<blockquote>
<p>  如按年分2010-2023, 再按月分1-12, 再按日分1-30, …</p>
</blockquote>
<h4 id="分区表的使用"><a href="#分区表的使用" class="headerlink" title="分区表的使用"></a>分区表的使用</h4><ul>
<li>基本语法</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">create table table_name(</span><br><span class="line">	数据列 列类型,</span><br><span class="line">    ...</span><br><span class="line">) partitioned by (分区列 列类型, ...)</span><br><span class="line">row fomat delitimed fields terminated by &#x27;&#x27;;</span><br></pre></td></tr></table></figure>

<ul>
<li>创建分区表(单分区)</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">create table score(</span><br><span class="line">    id string,</span><br><span class="line">    cid string,</span><br><span class="line">    score int</span><br><span class="line">) partitioned by (month string)</span><br><span class="line">row format delimited fields terminated by &#x27;\t&#x27;;</span><br></pre></td></tr></table></figure>

<ul>
<li>加载Linux本地数据到表中, 并指定分区列</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">load  data local inpath &#x27;/home/hadoop/score.txt&#x27; overwrite into table score partition(month=&#x27;202304&#x27;);</span><br><span class="line">select * from score;</span><br></pre></td></tr></table></figure>

<img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419150041.png" alt="image-20230419150033792" style="zoom:80%;" />

<p>数据列来源于创建表时的字段, 分区列来源于指定的分区, 各个分区的数据独立, 不会相互混淆</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419150120.png" alt="image-20230419150118101"></p>
<p>再执行一下加载数据操作, 设定<code>month=202303</code>, 此时有两个分区</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419150316.png" alt="image-20230419150315416"></p>
<ul>
<li>创建分区表(多分区)</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">create table score2(</span><br><span class="line">    id string,</span><br><span class="line">    cid string,</span><br><span class="line">    score int</span><br><span class="line">) partitioned by (year string, month string, day string)</span><br><span class="line">row format delimited fields terminated by &#x27;\t&#x27;;</span><br></pre></td></tr></table></figure>

<ul>
<li>加载数据</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">load  data local inpath &#x27;/home/hadoop/score.txt&#x27;</span><br><span class="line">    into table score2 partition(year=&#x27;2023&#x27;, month=&#x27;04&#x27;, day=&#x27;18&#x27;);</span><br><span class="line">select * from score2;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419151143.png" alt="image-20230419151141639"></p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419151256.png" alt="image-20230419151254693"></p>
<p>分区表可以极大提高特定场景下的Hive的操作性能(不用对全表进行操作)</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">select * from score2 where year=&#x27;2023&#x27; and month=&#x27;04&#x27; and day=&#x27;18&#x27;;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419151656.png" alt="image-20230419151655666"></p>
<h3 id="分桶表"><a href="#分桶表" class="headerlink" title="分桶表"></a>分桶表</h3><p>分桶和分区一样, 也是一种通过改变表的储存模式, 从而完成对表优化的一种调优方式</p>
<p>分区是将表拆分到<font color='cpink'>不同的子文件夹</font>中, 而分桶是将表拆分到<font color=cpink>固定数量的不同文件</font>中进行存储</p>
<h4 id="分桶表的创建"><a href="#分桶表的创建" class="headerlink" title="分桶表的创建"></a>分桶表的创建</h4><ul>
<li>开启分桶的自动优化( 自动匹配reduce task的数量和桶数量一致 )</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">set hive.enforce.bucketing=ture;</span><br></pre></td></tr></table></figure>

<ul>
<li>创建分桶表</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">create table  course(c_id string, c_name string, t_id string)</span><br><span class="line">clustered by (c_id) into 3 buckets </span><br><span class="line">row format delimited fields terminated by &#x27;\t&#x27;;</span><br></pre></td></tr></table></figure>

<ul>
<li>分桶表的数据加载</li>
</ul>
<p>桶表的数据加载, 由于桶表的数据加载不能通过<code>load data</code>的方式, 只能通过<code>insert select</code>, 所以可以采用:</p>
<ol>
<li>创建一个临时表(内部&#x2F;外部均可), 通过<code>load data</code>加载数据到表中</li>
</ol>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">create table  course_temp(c_id string, c_name string, t_id string)</span><br><span class="line">row format delimited fields terminated by &#x27;\t&#x27;;</span><br><span class="line">load data local inpath &#x27;/home/hadoop/course.txt&#x27; into table course_temp;</span><br><span class="line">select * from course_temp;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419181414.png" alt="image-20230419181410312"></p>
<ol>
<li>通过<code>insert select</code>从临时表向桶表中插入数据</li>
</ol>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">insert overwrite table course select * from course_temp cluster by (c_id);</span><br><span class="line">select * from course;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419181450.png" alt="image-20230419181448531"></p>
<p>在HDFS中, 查看数据文件</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419181540.png" alt="image-20230419181537715"></p>
<blockquote>
<p>  <font color=cpink>为什么不可以使用load data, 必须使用insert select插入数据</font></p>
</blockquote>
<p>如果没有分桶设置, 插入(加载)数据只是简单的将数据列放入:</p>
<ul>
<li>表的存储文件夹中(无分区)</li>
<li>表的指定分区文件夹中(有分区)</li>
</ul>
<p>一旦有了分桶的设置, 当数据插入时, 需要一分为n, 进入n个桶文件中</p>
<ul>
<li>数据划分的规则</li>
</ul>
<p>数据的n份基于<font color=cpink size=5>分桶列的值进行hash取模</font>来决定, 由于load data<font color=cpink>不会触发MapReduce, 也就是没有计算过程(无法执行hash算法)</font>, 只能简单移动数据, 所以无法用于分桶表的数据插入</p>
<ul>
<li>Hash取模</li>
</ul>
<p>Hash算法是一种数据加密算法, 同样的值被Hash加密后的结果是一致的.</p>
<p>假设分桶数为3, 即将Hash的结果对3取模(除以3取其余数), 可以得到以下三种情况</p>
<p><code>0   1   2</code></p>
<p>即, 无论任何数据 ,得到的取模结果均是: 0, 1, 2中的一个</p>
<p><font color=cpink size=5>所以必须使用insert select 的语法, 因为其会触发MapReduce, 进行取模计算. </font></p>
<h4 id="分桶表的性能提升"><a href="#分桶表的性能提升" class="headerlink" title="分桶表的性能提升"></a>分桶表的性能提升</h4><blockquote>
<p>  分区表的性能提升: 在指定的分区列的前提下, 减少被操作的数据量, 从而提升性能</p>
</blockquote>
<p>分桶表的性能提升: 基于分桶列的特定操作, 如: 过滤, JOIN, 分组, 均可带来性能提升</p>
<h3 id="修改表"><a href="#修改表" class="headerlink" title="修改表"></a>修改表</h3><ul>
<li>表重命名</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">alter table old_table_name to new_table_name;</span><br></pre></td></tr></table></figure>

<p>如:  <code>alter table score3 to score4;</code></p>
<ul>
<li>修改表属性值</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">alter table table_name set tblproperties table_properties;</span><br></pre></td></tr></table></figure>

<p><code>table_properties</code>: (property_name&#x3D;property_value, …)</p>
<p>如: <code>alter table table_name set tblproperties(&quot;EXTERNAL&quot;=&quot;TRUE&quot;);</code>修改内外部表属性</p>
<p>​	  <code>alter table table_name set tblproperties(&quot;COMMENT&quot;=&quot;NEW_COMMENT&quot;);</code>修改表注释</p>
<ul>
<li>添加分区</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">alter table table_name add partition(分区=&#x27;&#x27;);</span><br><span class="line"># 分区是空的没有数据, 需要手动添加或上传数据文件(创建了一个新的分区文件夹)</span><br></pre></td></tr></table></figure>

<ul>
<li>修改分区值</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">alter table table_name partition(分区=&#x27;原分区&#x27;) rename to partition(分区=&#x27;新分区&#x27;); </span><br></pre></td></tr></table></figure>

<ul>
<li>删除分区</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">alter table table_name drop partition(分区=&#x27;&#x27;);</span><br></pre></td></tr></table></figure>

<ul>
<li>添加列</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">alter table table_name add columns(v1 type, v2 type, ...);</span><br></pre></td></tr></table></figure>

<ul>
<li>修改列名</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">alter table table_name change v1 v1_new 原type</span><br></pre></td></tr></table></figure>

<ul>
<li>删除表</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">drop table table_name;</span><br></pre></td></tr></table></figure>

<ul>
<li>清空表</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">truncate table table_name;</span><br></pre></td></tr></table></figure>

<p>操作示例</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line">-- 1. 修改表名</span><br><span class="line">alter table score2 rename to score3;</span><br><span class="line">-- 2. 修改表属性值</span><br><span class="line">desc formatted score3;</span><br><span class="line">alter table score3 set tblproperties (&quot;EXTERNAL&quot;=&quot;TRUE&quot;);</span><br><span class="line">alter table score3 set tblproperties (&quot;COMMENT&quot;=&quot;this is comment&quot;);</span><br><span class="line">-- 3. 添加表的分区</span><br><span class="line">alter table score3 add partition (year=&#x27;2023&#x27;, month=&#x27;01&#x27;, day=&#x27;01&#x27;);</span><br><span class="line">-- 4. 修改分区值(修改元数据记录, 在HDFS中的实体文件夹不会改名, 但是在元数据记录中是改了名的) - 一般不建议更改</span><br><span class="line">alter table score3 partition (year=&#x27;2023&#x27;, month=&#x27;01&#x27;, day=&#x27;01&#x27;) rename to partition (year=&#x27;2023&#x27;, month=&#x27;01&#x27;, day=&#x27;16&#x27;);</span><br><span class="line">-- 5. 删除分区(只删除了元数据, 数据本身还在)</span><br><span class="line">alter table score3 drop partition (year=&#x27;2023&#x27;, month=&#x27;01&#x27;, day=&#x27;16&#x27;);</span><br><span class="line"></span><br><span class="line">-- 6. 添加列</span><br><span class="line">alter table score3 add columns (v1 int, v2 string);</span><br><span class="line">-- 7. 修改列名(不能该原本列的类型)</span><br><span class="line">alter table score3 change v1 v1_new int;</span><br><span class="line">-- 8. 清空表数据(只能清空内部表的数据)</span><br><span class="line">truncate table score3;</span><br><span class="line">-- 9. 删除表</span><br><span class="line">drop table score3</span><br></pre></td></tr></table></figure>

<h3 id="复杂类型操作"><a href="#复杂类型操作" class="headerlink" title="复杂类型操作"></a>复杂类型操作</h3><h4 id="Array-数组类型"><a href="#Array-数组类型" class="headerlink" title="Array 数组类型"></a>Array 数组类型</h4><blockquote>
<p>  在一个列中表示多个信息, 保存一组相同类型的元素</p>
</blockquote>
<p>创建一个Array数据表</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">create table test_array(name string, work_locations array&lt;string&gt;)</span><br><span class="line">row format delimited fields terminated by &#x27;\t&#x27;</span><br><span class="line">collection items terminated by &#x27;,&#x27;;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419193723.png" alt="image-20230419193721700"></p>
<p>通过load data加载数据</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">load data local inpath &#x27;/home/hadoop/data_for_array.txt&#x27; into table test_array;</span><br><span class="line">select * from test_array;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419194029.png" alt="image-20230419194027708"></p>
<p>其他使用:</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre></td><td class="code"><pre><span class="line">-- 查询每个人的第一个工作地点</span><br><span class="line">select name, work_locations[0] from test_array;</span><br><span class="line">-- 查询每个人工作地点个数</span><br><span class="line">select name, size(work_locations) from test_array;</span><br><span class="line">-- 查询哪一个在天津工作过(work_locations包含tianjin)</span><br><span class="line">select name from test_array where array_contains(work_locations, &#x27;tianjin&#x27;);</span><br></pre></td></tr></table></figure>

<h4 id="Map类型"><a href="#Map类型" class="headerlink" title="Map类型"></a>Map类型</h4><p>Map类型就是: key-value型数据格式, 如下数据文件显示, 其中<code>members</code>字段是key-value型数据, 字段与字段分隔符为’,’ ; map字段之间的分隔符为’#’ ; map内部k-v分隔符为 ‘:’</p>
<figure class="highlight text"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">id, name, members, age</span><br><span class="line">1, zhangsan, father:xiaoming#mother:xiaohong, 28</span><br><span class="line">2, lisi, father:libai#mother:damei, 22</span><br><span class="line">...</span><br></pre></td></tr></table></figure>

<ul>
<li>建表语句</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">create table test_map(</span><br><span class="line">	id int,</span><br><span class="line">    name string,</span><br><span class="line">    members map&lt;string, string&gt;,</span><br><span class="line">    age int</span><br><span class="line">)row format delimited fields terminated by &#x27;,&#x27;</span><br><span class="line">collection items terminated by &#x27;#&#x27;</span><br><span class="line">map keys terminated by &#x27;:&#x27;;</span><br></pre></td></tr></table></figure>

<ul>
<li>加载数据并显示</li>
</ul>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419200055.png" alt="image-20230419200054275"></p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">load data local inpath &#x27;/home/hadoop/data_for_map.txt&#x27; into table test_map;</span><br><span class="line">select * from test_map;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419200037.png" alt="image-20230419200036001"></p>
<p>其他使用:</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">-- 查看成员中 每个的父亲</span><br><span class="line">select id, name, members[&#x27;father&#x27;] from test_map t;</span><br><span class="line">-- 取出map中的全部key, 返回类型是array</span><br><span class="line">select map_keys(members) from test_map;</span><br><span class="line">-- 取出map中的全部value -&gt; array</span><br><span class="line">select map_values(members) from test_map;</span><br><span class="line">-- size查看map的元素中(k-v对)的个数</span><br><span class="line">select size(members) from test_map;</span><br><span class="line">-- array_contains查看指定的数据是否包含在map中, 是否有sister这个key</span><br><span class="line">select * from test_map where array_contains(map_keys(members), &#x27;sister&#x27;);</span><br><span class="line">-- array_contains查看指定的数据是否包含在map中, 是否有 王林 这个value</span><br><span class="line">select * from test_map where array_contains(map_values(members), &#x27;王林&#x27;);</span><br></pre></td></tr></table></figure>

<h4 id="Struct类型"><a href="#Struct类型" class="headerlink" title="Struct类型"></a>Struct类型</h4><p>Struct类型是一个复合类型, 可以在一个列中存入多个子列, 每个子列允许设置类型和名称</p>
<p>如下文件, 字段之间’#’分隔, struct之间’:’分隔</p>
<figure class="highlight text"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">1#周杰伦:11</span><br><span class="line">2#李锦记:16</span><br><span class="line">...</span><br></pre></td></tr></table></figure>

<ul>
<li>建表语句</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">create table test_struct(</span><br><span class="line">	id string,</span><br><span class="line">    info struct&lt;name:string, age:int&gt;</span><br><span class="line">) row format delimited fields terminated by &#x27;#&#x27;</span><br><span class="line">    COLLECTION ITEMS TERMINATED BY &#x27;:&#x27;;</span><br></pre></td></tr></table></figure>

<ul>
<li>加载数据并显示</li>
</ul>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419202128.png" alt="image-20230419202127057"></p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">load data local inpath &#x27;/home/hadoop/data_for_struct.txt&#x27; into table test_struct;</span><br><span class="line">select id, info from test_struct;</span><br><span class="line">-- 此处的info是一个object类型</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419202245.png" alt="image-20230419202243790"></p>
<p>可以通过如下方法得到具体信息值</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">select id, info.name, info.age from test_struct;</span><br></pre></td></tr></table></figure>

<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419202451.png" alt="image-20230419202449134"></p>
<h4 id="总结"><a href="#总结" class="headerlink" title="总结"></a>总结</h4><p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230419203111.png" alt="image-20230419203109714"></p>
<h3 id="数据查询"><a href="#数据查询" class="headerlink" title="数据查询"></a>数据查询</h3><h4 id="基本查询"><a href="#基本查询" class="headerlink" title="基本查询"></a>基本查询</h4><ul>
<li>创建订单表和用户表</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br></pre></td><td class="code"><pre><span class="line">create database itheima;</span><br><span class="line">use itheima;</span><br><span class="line">-- 订单表</span><br><span class="line">create table orders(</span><br><span class="line">    orderId bigint comment &#x27;订单id&#x27;,</span><br><span class="line">    orderNo string comment &#x27;订单编号&#x27;,</span><br><span class="line">    shopId bigint comment &#x27;门店id&#x27;,</span><br><span class="line">    userId bigint comment &#x27;用户id&#x27;,</span><br><span class="line">    orderStatus tinyint comment &#x27;订单状态&#x27;,</span><br><span class="line">    goodsMoney double comment &#x27;商品金额&#x27;,</span><br><span class="line">    deliverMoney double comment &#x27;运费&#x27;,</span><br><span class="line">    totalMoney double comment &#x27;订单金额&#x27;,</span><br><span class="line">    realTotalMoney double comment &#x27;订单实际支付金额&#x27;,</span><br><span class="line">    payType tinyint comment &#x27;支付方式&#x27;,</span><br><span class="line">    isPay tinyint comment &#x27;是否支付&#x27;,</span><br><span class="line">    userName string comment &#x27;收件人姓名&#x27;,</span><br><span class="line">    userAddress string comment &#x27;收件人地址&#x27;,</span><br><span class="line">    userPhone string comment  &#x27;收件人电话&#x27;,</span><br><span class="line">    createTime timestamp comment &#x27;下单时间&#x27;,</span><br><span class="line">    payTime timestamp comment &#x27;支付时间&#x27;,</span><br><span class="line">    totalPayFee int comment &#x27;总支付金额&#x27;</span><br><span class="line">) row format delimited fields terminated by &#x27;\t&#x27;;</span><br><span class="line"></span><br><span class="line">load data local inpath &#x27;/home/hadoop/itheima_orders.txt&#x27; into table orders;</span><br><span class="line"></span><br><span class="line">-- 用户表</span><br><span class="line">create table users(</span><br><span class="line">    userId int,</span><br><span class="line">    loginName string,</span><br><span class="line">    loginSecret int,</span><br><span class="line">    loginPwd string,</span><br><span class="line">    userSex tinyint,</span><br><span class="line">    userName string,</span><br><span class="line">    trueName string,</span><br><span class="line">    birthday date,</span><br><span class="line">    userPhone string,</span><br><span class="line">    userQQ string,</span><br><span class="line">    userScore int,</span><br><span class="line">    userTotalScore int,</span><br><span class="line">    userFrom tinyint,</span><br><span class="line">    userMoney double,</span><br><span class="line">    lockMoney double,</span><br><span class="line">    createTime timestamp,</span><br><span class="line">    payPwd string,</span><br><span class="line">    rechargeMoney double</span><br><span class="line">) row format delimited fields terminated by &#x27;\t&#x27;;</span><br><span class="line">load data local inpath &#x27;/home/hadoop/itheima_users.txt&#x27; into table users;</span><br></pre></td></tr></table></figure>

<ul>
<li>查询示例</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre></td><td class="code"><pre><span class="line">-- 查询所有</span><br><span class="line">select * from orders;</span><br><span class="line">-- 查询单列</span><br><span class="line">select orderId, totalmoney, username, useraddress, paytime from orders;</span><br><span class="line">-- 查询数据量</span><br><span class="line">select count(*) from orders;</span><br><span class="line">-- 过滤广东订单</span><br><span class="line">select * from orders where useraddress like &#x27;%广东%&#x27;;</span><br><span class="line">-- 找出广东省单笔营业额最大的订单</span><br><span class="line">select * from orders where userAddress like &#x27;%广东%&#x27; order by totalMoney desc limit 1;</span><br><span class="line">-- 统计未支付, 已支付各自的人数</span><br><span class="line">select isPay, count(*) as cnt from orders group by isPay;</span><br><span class="line">-- 在已支付的订单中, 统计每一个用户最高的一笔消费额</span><br><span class="line">select userid, max(totalMoney) as max_money from orders where isPay == 1 group by userid;</span><br><span class="line">-- 统计每一个用户订单的平均消费额</span><br><span class="line">select userid, avg(totalMoney) as avg_money from orders group by userid;</span><br><span class="line">-- 统计每一个用户订单的平均消费额, 并过滤大于10000的数据</span><br><span class="line">select userid, avg(totalMoney) as avg_money from orders group by userid having avg_money&gt;10000;</span><br><span class="line">-- join订单表和用户表, 找出用户username</span><br><span class="line">select o.orderId, o.userId, u.username, o.totalMoney, o.userAddress from orders o join itheima.users u on o.userId = u.userid;</span><br><span class="line">select o.orderId, o.userId, u.username, o.totalMoney, o.userAddress from orders o left join itheima.users u on o.userId = u.userid;</span><br></pre></td></tr></table></figure>

<h4 id="RLIKE正则匹配"><a href="#RLIKE正则匹配" class="headerlink" title="RLIKE正则匹配"></a>RLIKE正则匹配</h4><blockquote>
<p>  正则匹配</p>
</blockquote>
<p>正则表达式是一种规则集合, 通过特定的规则字符描述, 来判断字符串是否符合规则</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230421204313.png" alt="image-20230421204310810"></p>
<p>Hive 提供RLIKE关键字, 可以供用户使用正则和数据进行匹配</p>
<ul>
<li>示例</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">-- 查找广东省的数据</span><br><span class="line">select * from orders where userAddress rlike &#x27;.*广东*.&#x27;;</span><br><span class="line">-- 查找符合地址为 xx省 xx市 xx区的数据</span><br><span class="line">select * from orders where userAddress rlike &#x27;..省 ..市 ..区&#x27;;</span><br><span class="line">-- 查找用户姓为张,王,邓的数据</span><br><span class="line">select * from orders where userName rlike &#x27;[张王邓]\\S+&#x27;;</span><br><span class="line">-- 查找手机号符合188****0***规则</span><br><span class="line">select * from orders where userPhone rlike &#x27;188\\S&#123;4&#125;0\\S&#123;3&#125;&#x27;;</span><br></pre></td></tr></table></figure>

<h4 id="UNION联合"><a href="#UNION联合" class="headerlink" title="UNION联合"></a>UNION联合</h4><p>UNION用于将多个SELECT语句的结果合成单个结果集, 每个SELECT语句返回的列的数量和名称必须相同, 否则将引发架构错误</p>
<ul>
<li>示例</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br></pre></td><td class="code"><pre><span class="line">-- union联合</span><br><span class="line">create table course(</span><br><span class="line">    c_id string,</span><br><span class="line">    c_name string,</span><br><span class="line">    t_id string</span><br><span class="line">) row format delimited fields terminated by &#x27;\t&#x27;;</span><br><span class="line">load data local inpath &#x27;/home/hadoop/course.txt&#x27; into table course;</span><br><span class="line">-- 联合两个查询结果集[默认去重]</span><br><span class="line">select * from course where t_id == &#x27;周杰轮&#x27;</span><br><span class="line">    union</span><br><span class="line">select * from course where t_id == &#x27;王力鸿&#x27;;</span><br><span class="line">-- 不去重</span><br><span class="line">select * from course</span><br><span class="line">    union all</span><br><span class="line">select * from course;</span><br><span class="line">-- 其他用法</span><br><span class="line">-- 1. union写在from中, 在子查询中</span><br><span class="line">select t_id, count(*) from (</span><br><span class="line">    select t_id from course where t_id == &#x27;周杰轮&#x27;</span><br><span class="line">            union all</span><br><span class="line">    select t_id from course where t_id == &#x27;王力鸿&#x27;</span><br><span class="line">    ) as u group by t_id;</span><br><span class="line">-- 2. insert select中</span><br><span class="line">create table course2(</span><br><span class="line">    c_id string,</span><br><span class="line">    c_name string,</span><br><span class="line">    t_id string</span><br><span class="line">) row format delimited fields terminated by &#x27;\t&#x27;;</span><br><span class="line">insert overwrite table course2</span><br><span class="line">select * from course</span><br><span class="line">union</span><br><span class="line">select * from course;</span><br></pre></td></tr></table></figure>

<h4 id="数据采样"><a href="#数据采样" class="headerlink" title="数据采样"></a>数据采样</h4><p>Hive提供快速抽样的语法, 可以快速从大表中随机抽取一些数据供用户查看</p>
<p>进行随机抽样, 本质上就是使用tablesample函数</p>
<ul>
<li>基于随机分桶抽样</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">select ... from tbl tablesample(bucket x out of y on (colname | rand()))</span><br></pre></td></tr></table></figure>

<blockquote>
<p>  y 表示将表数据随机划分为y份</p>
<p>  x 表示从y里面随机抽取x份数据作为取样</p>
<p>  colname 表示随机的依据基于某个列的值</p>
<p>  rand() 表示随机的依据基于整行</p>
</blockquote>
<ul>
<li>示例</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">-- 以username为分桶依据</span><br><span class="line">select username, userid, totalmoney from orders tablesample ( bucket 3 out of 10 on username);</span><br><span class="line">-- 完全随机, 每一次随机分桶</span><br><span class="line">select username, userid, totalmoney from orders tablesample ( bucket 3 out of 10 on rand());</span><br></pre></td></tr></table></figure>

<ul>
<li>基于数据块抽样</li>
</ul>
<p>这种语法进行抽样时, 如果条件不变, 每一次抽样的结果都一致, 只是按照数据顺序从前往后取.</p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">select ... from tbl tablesample(num rows|num percent|num(K|M|G));</span><br></pre></td></tr></table></figure>

<blockquote>
<p>  num rows 表示抽样num条数据</p>
<p>  num percent 表示抽样num百分比的数据</p>
<p>  num(K|M|G) 表示抽取num大小的数据, 单位可以是K,M,G，表示KB, MB, GB</p>
</blockquote>
<ul>
<li>示例</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre></td><td class="code"><pre><span class="line">-- 抽样[数据块], 按顺序抽取, 每次条件不变, 抽取结果不变</span><br><span class="line">-- 取100条</span><br><span class="line">select * from orders tablesample ( 100 rows );</span><br><span class="line">-- 取1%数据</span><br><span class="line">select * from orders tablesample ( 1 percent );</span><br><span class="line">-- 取1KB数据[无法运行]</span><br><span class="line">select * from orders tablesample( 1K );</span><br></pre></td></tr></table></figure>

<h4 id="虚拟列"><a href="#虚拟列" class="headerlink" title="虚拟列"></a>虚拟列</h4><p>Virtual Columns虚拟列</p>
<p>虚拟列是Hive内置的可以在查询语句中使用的特殊标记, 可以查询数据本身的详细参数</p>
<p>Hive目前可以用3个虚拟列:</p>
<ul>
<li><code>INPUT_FILE_NAME</code> 显示数据行所在的具体文件</li>
<li><code>BLOCK_OFFSET_INSIDE_FILE</code> 显示数据行所在文件的偏移量</li>
<li><code>ROW_OFFSET_INSIDE_BLOCK</code> 显示数据所在HDFS块的偏移量<ul>
<li>此虚拟列需要设置: <code>set hive.exec.rowoffset=true</code> 才可以使用</li>
</ul>
</li>
</ul>
<p>作用:</p>
<ul>
<li>查看行级别的数据详细参数</li>
<li>可以用于<code>where</code>, <code>group by</code> 等各类统计计算中</li>
<li>可以协助进行错误排查工作</li>
</ul>
<h3 id="函数"><a href="#函数" class="headerlink" title="函数"></a>函数</h3><h4 id="函数分类"><a href="#函数分类" class="headerlink" title="函数分类"></a>函数分类</h4><p>Hive的函数分为两大类: <font color=cpink>位置函数</font>(Built-in Functions), <font color=cpink>用户定义函数</font>(User-Defined Functions)</p>
<p><img src="https://gitee-blog-1317332932.cos.ap-nanjing.myqcloud.com/20230422092918.png" alt="image-20230422092916955"></p>
<p>使用<code>show functions</code> 查看当下可用的所有函数</p>
<p>通过<code>describe function extended funcname</code> 查看函数的使用方法</p>
<ul>
<li>数学函数 Mathmatical Functions</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">-- 取整函数, 返回double类型的整数值部分, 四舍五入</span><br><span class="line">select round(3.144);</span><br><span class="line">-- 指定精度取整</span><br><span class="line">select round(3.156, 2);</span><br><span class="line">-- 取随机数函数, 返回0-1之间的随机数</span><br><span class="line">select rand();</span><br><span class="line">-- 设置随机数种子, 设置种子后, 每次运行结果相同</span><br><span class="line">select rand(4);</span><br><span class="line">-- 取绝对值</span><br><span class="line">select abs(-3);</span><br><span class="line">-- 求pi</span><br><span class="line">select pi();</span><br></pre></td></tr></table></figure>

<ul>
<li>集合函数</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre></td><td class="code"><pre><span class="line">-- 求元素个数</span><br><span class="line">select size(work_locations) from myhive.test_array;</span><br><span class="line">-- 取出map所有的key</span><br><span class="line">select map_keys(members) from myhive.test_map;</span><br><span class="line">-- 取出map所有的value</span><br><span class="line">select map_values(members) from myhive.test_map;</span><br><span class="line">-- 对map进行自然排序</span><br><span class="line">select *, sort_array(work_locations) from myhive.test_array;</span><br></pre></td></tr></table></figure>

<ul>
<li>类型转换函数</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><span class="line">-- 转换为二进制</span><br><span class="line">select binary (&#x27;hadoop&#x27;);</span><br><span class="line">-- 自由转换, 类型转换失败则报错或返回NULL</span><br><span class="line">select cast(&#x27;1&#x27; as bigint);</span><br></pre></td></tr></table></figure>

<ul>
<li>日期函数</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre></td><td class="code"><pre><span class="line">-- 当前时间戳</span><br><span class="line">select current_timestamp();</span><br><span class="line">-- 当前日期</span><br><span class="line">select current_date();</span><br><span class="line">-- 时间戳转日期</span><br><span class="line">select to_date(current_timestamp());</span><br><span class="line">-- 年,季度,月,日等</span><br><span class="line">select year(&#x27;2022-04-02&#x27;);</span><br><span class="line">select quarter(&#x27;2022-04-02&#x27;);</span><br><span class="line">select month(&#x27;2022-04-02&#x27;);</span><br><span class="line">select day(&#x27;2022-04-02&#x27;);</span><br><span class="line">select hour(&#x27;2022-04-02 10:23:45&#x27;);</span><br><span class="line">select minute(&#x27;2022-04-02 10:23:45&#x27;);</span><br><span class="line">select second(&#x27;2022-04-02 10:23:45&#x27;);</span><br><span class="line">select weekofyear(&#x27;2022-04-02 10:23:45&#x27;);</span><br><span class="line">-- 日期之间的天数</span><br><span class="line">select datediff(&#x27;2022-04-02&#x27;, &#x27;2019-04-22&#x27;);</span><br><span class="line">-- 日期相加,相减</span><br><span class="line">select date_add(&#x27;2020-02-28&#x27;, 2);</span><br><span class="line">select date_sub(&#x27;2021-03-01&#x27;, 1);</span><br></pre></td></tr></table></figure>

<ul>
<li>条件函数</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre></td><td class="code"><pre><span class="line">-- if判断, 如果ture 返回第一个参数, false 返回第二个参数</span><br><span class="line">select if(truename == &#x27;萧呀轩&#x27;, &#x27;xxxxx&#x27;, truename) from users;</span><br><span class="line">-- null 判断</span><br><span class="line">select isnull(truename) from users;</span><br><span class="line">-- not null 判断</span><br><span class="line">select isnotnull(truename) from users;</span><br><span class="line">-- nvl, 如果value为null, 则返回default_value, 否则返回value</span><br><span class="line">select nvl(truename, &#x27;none&#x27;) from users;</span><br><span class="line">-- 返回第一个不是null的value, 如果所有的value都是null, 则返回null</span><br><span class="line">select coalesce(truename, birthday) from users;</span><br><span class="line">-- 当a == b 时, 返回c; 当a == d 时, 返回e; ... 否则返回f</span><br><span class="line">select case username when &#x27;萧呀轩&#x27; then &#x27;xyx&#x27; when &#x27;周杰轮&#x27; then &#x27;zjl&#x27; else &#x27;xxx&#x27; end  from users;</span><br><span class="line">-- 当a == true, 返回b;当c == true, 返回d, ... 否则返回f</span><br><span class="line">select case when truename == &#x27;周杰轮&#x27; then &#x27;zjl&#x27; else userName end from users;</span><br><span class="line">-- 当a == b, 则返回null, 否则返回a</span><br><span class="line">select userName, nullif(userName, &#x27;周杰轮&#x27;) from users;</span><br><span class="line">-- 如果参数为False, 则报错</span><br><span class="line">select assert_true(1&gt;2);</span><br></pre></td></tr></table></figure>

<ul>
<li>字符串函数</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre></td><td class="code"><pre><span class="line">-- 连接字符串 _ws : 可指定分隔符</span><br><span class="line">select concat(userName, loginName) from users;</span><br><span class="line">select concat_ws(&#x27;,&#x27;, userName, loginName) from users;</span><br><span class="line">-- 统计字符串长度</span><br><span class="line">select length(userName)from users;</span><br><span class="line">-- 转大小写</span><br><span class="line">select lower(&#x27;ABCD&#x27;);</span><br><span class="line">select upper(&#x27;abcd&#x27;);</span><br><span class="line">-- 去除首位空格</span><br><span class="line">select trim(&#x27;   hello world   &#x27;);</span><br><span class="line">-- 字符串分隔</span><br><span class="line">select split(&#x27;hadoop,bigdata,hive,spark&#x27;, &#x27;,&#x27;);</span><br></pre></td></tr></table></figure>

<ul>
<li>数据脱敏函数</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">-- hash加密(结果是16进制字符串)</span><br><span class="line">select mask_hash(&#x27;hadoop&#x27;);</span><br></pre></td></tr></table></figure>

<ul>
<li>其他函数</li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line">-- hash加密(结果是数字)</span><br><span class="line">select hash(&#x27;hadoop&#x27;);</span><br><span class="line">-- 当前登陆用户</span><br><span class="line">select current_user();</span><br><span class="line">-- 当前数据库</span><br><span class="line">select current_database();</span><br><span class="line">-- hive版本</span><br><span class="line">select version();</span><br><span class="line">-- 返回指定参数的md5值</span><br><span class="line">select md5(&#x27;hadoop&#x27;);</span><br></pre></td></tr></table></figure>



</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta">文章作者: </span><span class="post-copyright-info"><a href="https://crappier.github.io">fanhao</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta">文章链接: </span><span class="post-copyright-info"><a href="https://crappier.github.io/2023/04/13/%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%85%A5%E9%97%A8/">https://crappier.github.io/2023/04/13/%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%85%A5%E9%97%A8/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta">版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="https://crappier.github.io" target="_blank">Depicter</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/tags/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></div><div class="post_share"><div class="social-share" data-image="/img/cover/cover_4.jpg" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/butterfly-extsrc/sharejs/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/sharejs/dist/js/social-share.min.js" defer></script></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/2023/05/16/Go%E8%AF%AD%E8%A8%80%E5%AD%A6%E4%B9%A0/" title="Go语言学习"><img class="cover" src="/img/cover/cover_4.jpg" onerror="onerror=null;src='/img/404.jpg'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">Go语言学习</div></div></a></div><div class="next-post pull-right"><a href="/2023/04/10/Python%E7%9A%84%E5%9E%83%E5%9C%BE%E5%9B%9E%E6%94%B6%E6%9C%BA%E5%88%B6/" title="Python的垃圾回收机制"><img class="cover" src="/img/cover/cover_1.jpg" onerror="onerror=null;src='/img/404.jpg'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">Python的垃圾回收机制</div></div></a></div></nav></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="/img/avatar.jpg" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">fanhao</div><div class="author-info__description"></div></div><div class="card-info-data site-data is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">10</div></a><a href="/tags/"><div class="headline">标签</div><div class="length-num">5</div></a><a href="/categories/"><div class="headline">分类</div><div class="length-num">0</div></a></div><a id="card-info-btn" target="_blank" rel="noopener" href="https://gitee.com/fanhao0416"><i class="fab fa-gitee"></i><span>联系我</span></a><div class="card-info-social-icons is-center"><a class="social-icon" href="https://gitee.com/fanhao0416" target="_blank" title="Gitee"><i class="fab fa-git" style="color: #DC143C;"></i></a><a class="social-icon" href="mailto:2039216364@qq.com" target="_blank" title="Email"><i class="fas fa-envelope" style="color: #4a7dbe;"></i></a><a class="social-icon" href="http://wpa.qq.com/msgrd?v=3&amp;uin=2039216364&amp;site=qq&amp;menu=yes" target="_blank" title="QQ"><i class="fab fa-qq" style="color: #4a7dbe;"></i></a></div></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn fa-shake"></i><span>公告</span></div><div class="announcement_content">沧海易幻莫同，世人皆平庸</div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span><span class="toc-percentage"></span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#%E5%A4%A7%E6%95%B0%E6%8D%AE%E5%85%A5%E9%97%A8"><span class="toc-number">1.</span> <span class="toc-text">大数据入门</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%A4%A7%E6%95%B0%E6%8D%AE%E7%9A%84%E6%A0%B8%E5%BF%83%E5%B7%A5%E4%BD%9C%E4%BD%93%E7%B3%BB"><span class="toc-number">1.1.</span> <span class="toc-text">大数据的核心工作体系</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%A4%A7%E6%95%B0%E6%8D%AE%E8%BD%AF%E4%BB%B6%E7%94%9F%E6%80%81"><span class="toc-number">1.1.1.</span> <span class="toc-text">大数据软件生态</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BB%80%E4%B9%88%E6%98%AFHadoop"><span class="toc-number">1.1.2.</span> <span class="toc-text">什么是Hadoop</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Hadoop%E7%9A%84%E5%8F%91%E5%B1%95"><span class="toc-number">1.1.3.</span> <span class="toc-text">Hadoop的发展</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#HDFS"><span class="toc-number">1.2.</span> <span class="toc-text">HDFS</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%88%86%E5%B8%83%E5%BC%8F%E5%AD%98%E5%82%A8"><span class="toc-number">1.2.1.</span> <span class="toc-text">分布式存储</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E4%B8%BB%E4%BB%8E%E6%A8%A1%E5%BC%8F"><span class="toc-number">1.2.1.1.</span> <span class="toc-text">主从模式</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#HDFS%E7%9A%84%E5%9F%BA%E7%A1%80%E6%9E%B6%E6%9E%84"><span class="toc-number">1.2.2.</span> <span class="toc-text">HDFS的基础架构</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#HDFS%E9%9B%86%E7%BE%A4%E7%8E%AF%E5%A2%83%E9%83%A8%E7%BD%B2"><span class="toc-number">1.2.3.</span> <span class="toc-text">HDFS集群环境部署</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#VMware%E8%BD%AF%E4%BB%B6%E5%88%9B%E5%BB%BA%E8%99%9A%E6%8B%9F%E6%9C%BA%E6%9C%8D%E5%8A%A1%E5%99%A8%E5%B9%B6%E5%88%9B%E5%BB%BAhadoop%E7%94%A8%E6%88%B7"><span class="toc-number">1.2.3.1.</span> <span class="toc-text">VMware软件创建虚拟机服务器并创建hadoop用户</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%9C%A8VMware%E8%99%9A%E6%8B%9F%E6%9C%BA%E4%B8%8A%E9%83%A8%E7%BD%B2HDFS%E9%9B%86%E7%BE%A4"><span class="toc-number">1.2.3.2.</span> <span class="toc-text">在VMware虚拟机上部署HDFS集群</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#HDFS%E7%9A%84Shell%E6%93%8D%E4%BD%9C"><span class="toc-number">1.2.3.3.</span> <span class="toc-text">HDFS的Shell操作</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#%E8%BF%9B%E7%A8%8B%E5%90%AF%E5%81%9C%E7%AE%A1%E7%90%86"><span class="toc-number">1.2.3.3.1.</span> <span class="toc-text">进程启停管理</span></a><ol class="toc-child"><li class="toc-item toc-level-6"><a class="toc-link" href="#%E4%B8%80%E9%94%AE%E5%90%AF%E5%81%9C%E8%84%9A%E6%9C%AC"><span class="toc-number">1.2.3.3.1.1.</span> <span class="toc-text">一键启停脚本</span></a></li><li class="toc-item toc-level-6"><a class="toc-link" href="#%E5%8D%95%E8%BF%9B%E7%A8%8B%E5%90%AF%E5%81%9C"><span class="toc-number">1.2.3.3.1.2.</span> <span class="toc-text">单进程启停</span></a></li></ol></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%E6%93%8D%E4%BD%9C%E5%91%BD%E4%BB%A4"><span class="toc-number">1.2.3.3.2.</span> <span class="toc-text">文件系统操作命令</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#HDFS-WEB%E6%B5%8F%E8%A7%88"><span class="toc-number">1.2.3.3.3.</span> <span class="toc-text">HDFS WEB浏览</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#HDFS%E5%AE%A2%E6%88%B7%E7%AB%AF-Jetbrains%E4%BA%A7%E5%93%81%E6%8F%92%E4%BB%B6"><span class="toc-number">1.2.3.3.4.</span> <span class="toc-text">HDFS客户端-Jetbrains产品插件</span></a></li></ol></li><li class="toc-item toc-level-4"><a class="toc-link" href="#HDFS%E5%AD%98%E5%82%A8%E5%8E%9F%E7%90%86"><span class="toc-number">1.2.3.4.</span> <span class="toc-text">HDFS存储原理</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E5%A6%82%E4%BD%95%E5%9C%A8HDFS%E4%B8%AD%E5%AD%98%E5%82%A8%E7%9A%84"><span class="toc-number">1.2.3.4.1.</span> <span class="toc-text">数据如何在HDFS中存储的</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E5%9C%A8HDFS%E4%B8%AD%E5%A6%82%E4%BD%95%E4%BF%9D%E8%AF%81%E5%AE%89%E5%85%A8%E7%9A%84"><span class="toc-number">1.2.3.4.2.</span> <span class="toc-text">数据在HDFS中如何保证安全的</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#HDFS%E5%89%AF%E6%9C%AC%E5%9D%97%E6%95%B0%E9%87%8F%E7%9A%84%E9%85%8D%E7%BD%AE"><span class="toc-number">1.2.3.4.3.</span> <span class="toc-text">HDFS副本块数量的配置</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#NameNode%E5%85%83%E6%95%B0%E6%8D%AE"><span class="toc-number">1.2.3.4.4.</span> <span class="toc-text">NameNode元数据</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#HDFS%E6%95%B0%E6%8D%AE%E7%9A%84%E8%AF%BB%E5%86%99%E6%B5%81%E7%A8%8B"><span class="toc-number">1.2.3.4.5.</span> <span class="toc-text">HDFS数据的读写流程</span></a></li></ol></li></ol></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E5%88%86%E5%B8%83%E5%BC%8F%E8%AE%A1%E7%AE%97"><span class="toc-number">1.3.</span> <span class="toc-text">分布式计算</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%88%86%E5%B8%83%E5%BC%8F%E8%AE%A1%E7%AE%97%E6%A6%82%E8%BF%B0"><span class="toc-number">1.3.1.</span> <span class="toc-text">分布式计算概述</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%88%86%E6%95%A3-gt-%E6%B1%87%E6%80%BB%E6%A8%A1%E5%BC%8F-MapReduce"><span class="toc-number">1.3.1.1.</span> <span class="toc-text">分散-&gt;汇总模式(MapReduce)</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E4%B8%AD%E5%BF%83%E8%B0%83%E5%BA%A6-gt-%E6%AD%A5%E9%AA%A4%E6%89%A7%E8%A1%8C%E6%A8%A1%E5%BC%8F-Spark-Flink"><span class="toc-number">1.3.1.2.</span> <span class="toc-text">中心调度-&gt;步骤执行模式(Spark, Flink)</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#MapReduce%E6%A6%82%E8%BF%B0"><span class="toc-number">1.3.2.</span> <span class="toc-text">MapReduce概述</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#YARN%E7%9A%84%E6%A6%82%E8%BF%B0"><span class="toc-number">1.3.3.</span> <span class="toc-text">YARN的概述</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6"><span class="toc-number">1.3.3.1.</span> <span class="toc-text">资源调度</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#YARN%E7%9A%84%E8%B5%84%E6%BA%90%E8%B0%83%E5%BA%A6"><span class="toc-number">1.3.3.2.</span> <span class="toc-text">YARN的资源调度</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#YARN%E5%92%8CMapReduce%E7%9A%84%E5%85%B3%E7%B3%BB"><span class="toc-number">1.3.3.3.</span> <span class="toc-text">YARN和MapReduce的关系</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#YARN%E6%9E%B6%E6%9E%84-M-S%E6%9E%B6%E6%9E%84"><span class="toc-number">1.3.3.4.</span> <span class="toc-text">YARN架构(M-S架构)</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#YARN%E8%BE%85%E5%8A%A9%E8%A7%92%E8%89%B2"><span class="toc-number">1.3.3.5.</span> <span class="toc-text">YARN辅助角色</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#%E4%BB%A3%E7%90%86%E6%9C%8D%E5%8A%A1%E5%99%A8-ProxyServer"><span class="toc-number">1.3.3.5.1.</span> <span class="toc-text">代理服务器(ProxyServer)</span></a></li><li class="toc-item toc-level-5"><a class="toc-link" href="#%E5%8E%86%E5%8F%B2%E6%9C%8D%E5%8A%A1%E5%99%A8-JobHistoryServer"><span class="toc-number">1.3.3.5.2.</span> <span class="toc-text">历史服务器(JobHistoryServer)</span></a></li></ol></li><li class="toc-item toc-level-4"><a class="toc-link" href="#YARN%E9%83%A8%E7%BD%B2"><span class="toc-number">1.3.3.6.</span> <span class="toc-text">YARN部署</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E4%BD%93%E9%AA%8C-YARN"><span class="toc-number">1.3.3.7.</span> <span class="toc-text">体验: YARN</span></a><ol class="toc-child"><li class="toc-item toc-level-5"><a class="toc-link" href="#%E6%8F%90%E4%BA%A4MapReduce%E7%A8%8B%E5%BA%8F%E8%87%B3YARN%E8%BF%90%E8%A1%8C"><span class="toc-number">1.3.3.7.1.</span> <span class="toc-text">提交MapReduce程序至YARN运行</span></a><ol class="toc-child"><li class="toc-item toc-level-6"><a class="toc-link" href="#%E6%8F%90%E4%BA%A4wordcount%E7%A4%BA%E4%BE%8B%E7%A8%8B%E5%BA%8F"><span class="toc-number">1.3.3.7.1.1.</span> <span class="toc-text">提交wordcount示例程序</span></a></li><li class="toc-item toc-level-6"><a class="toc-link" href="#%E6%8F%90%E4%BA%A4%E6%B1%82%E5%9C%86%E5%91%A8%E7%8E%87%E7%A4%BA%E4%BE%8B%E7%A8%8B%E5%BA%8F"><span class="toc-number">1.3.3.7.1.2.</span> <span class="toc-text">提交求圆周率示例程序</span></a></li></ol></li></ol></li></ol></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Apache-Hive-%E5%85%A5%E9%97%A8"><span class="toc-number">1.4.</span> <span class="toc-text">Apache Hive(入门)</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#Hive%E6%9E%B6%E6%9E%84"><span class="toc-number">1.4.1.</span> <span class="toc-text">Hive架构</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#VMware%E9%83%A8%E7%BD%B2Hive"><span class="toc-number">1.4.2.</span> <span class="toc-text">VMware部署Hive</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Hive%E4%BD%93%E9%AA%8C"><span class="toc-number">1.4.3.</span> <span class="toc-text">Hive体验</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#HiveServer2-amp-Beeline"><span class="toc-number">1.4.4.</span> <span class="toc-text">HiveServer2 &amp; Beeline</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%90%AF%E5%8A%A8HiveServer2"><span class="toc-number">1.4.4.1.</span> <span class="toc-text">启动HiveServer2</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#beeline"><span class="toc-number">1.4.4.2.</span> <span class="toc-text">beeline</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#HiveServer-amp-PyCharm"><span class="toc-number">1.4.5.</span> <span class="toc-text">HiveServer &amp; PyCharm</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#Apache-Hive-%E4%BD%BF%E7%94%A8"><span class="toc-number">1.5.</span> <span class="toc-text">Apache Hive(使用)</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E5%BA%93%E6%93%8D%E4%BD%9C%E8%AF%AD%E6%B3%95"><span class="toc-number">1.5.1.</span> <span class="toc-text">数据库操作语法</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E5%BA%93%E6%93%8D%E4%BD%9C"><span class="toc-number">1.5.1.1.</span> <span class="toc-text">数据库操作</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E8%A1%A8%E6%93%8D%E4%BD%9C%E8%AF%AD%E6%B3%95"><span class="toc-number">1.5.2.</span> <span class="toc-text">数据表操作语法</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%88%9B%E5%BB%BA%E8%A1%A8%E7%9A%84%E8%AF%AD%E6%B3%95"><span class="toc-number">1.5.3.</span> <span class="toc-text">创建表的语法</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#Hive%E7%9A%84%E8%A1%A8%E7%B1%BB%E5%9E%8B"><span class="toc-number">1.5.4.</span> <span class="toc-text">Hive的表类型</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E5%8A%A0%E8%BD%BD%E5%92%8C%E5%AF%BC%E5%87%BA"><span class="toc-number">1.5.5.</span> <span class="toc-text">数据加载和导出</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E5%8A%A0%E8%BD%BD-LOAD%E8%AF%AD%E6%B3%95"><span class="toc-number">1.5.5.1.</span> <span class="toc-text">数据加载-LOAD语法</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E5%8A%A0%E8%BD%BD-INSERT-SELECT%E8%AF%AD%E6%B3%95"><span class="toc-number">1.5.5.2.</span> <span class="toc-text">数据加载-INSERT SELECT语法</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E5%AF%BC%E5%87%BA-INSERT-OVERWRITE%E6%96%B9%E5%BC%8F"><span class="toc-number">1.5.5.3.</span> <span class="toc-text">数据导出-INSERT OVERWRITE方式</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#Hive%E8%A1%A8%E6%95%B0%E6%8D%AE%E5%AF%BC%E5%87%BA-hive-shell"><span class="toc-number">1.5.5.4.</span> <span class="toc-text">Hive表数据导出-hive shell</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%88%86%E5%8C%BA%E8%A1%A8"><span class="toc-number">1.5.6.</span> <span class="toc-text">分区表</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%88%86%E5%8C%BA%E8%A1%A8%E7%9A%84%E4%BD%BF%E7%94%A8"><span class="toc-number">1.5.6.1.</span> <span class="toc-text">分区表的使用</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%88%86%E6%A1%B6%E8%A1%A8"><span class="toc-number">1.5.7.</span> <span class="toc-text">分桶表</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%88%86%E6%A1%B6%E8%A1%A8%E7%9A%84%E5%88%9B%E5%BB%BA"><span class="toc-number">1.5.7.1.</span> <span class="toc-text">分桶表的创建</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%88%86%E6%A1%B6%E8%A1%A8%E7%9A%84%E6%80%A7%E8%83%BD%E6%8F%90%E5%8D%87"><span class="toc-number">1.5.7.2.</span> <span class="toc-text">分桶表的性能提升</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BF%AE%E6%94%B9%E8%A1%A8"><span class="toc-number">1.5.8.</span> <span class="toc-text">修改表</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%A4%8D%E6%9D%82%E7%B1%BB%E5%9E%8B%E6%93%8D%E4%BD%9C"><span class="toc-number">1.5.9.</span> <span class="toc-text">复杂类型操作</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#Array-%E6%95%B0%E7%BB%84%E7%B1%BB%E5%9E%8B"><span class="toc-number">1.5.9.1.</span> <span class="toc-text">Array 数组类型</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#Map%E7%B1%BB%E5%9E%8B"><span class="toc-number">1.5.9.2.</span> <span class="toc-text">Map类型</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#Struct%E7%B1%BB%E5%9E%8B"><span class="toc-number">1.5.9.3.</span> <span class="toc-text">Struct类型</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%80%BB%E7%BB%93"><span class="toc-number">1.5.9.4.</span> <span class="toc-text">总结</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E6%9F%A5%E8%AF%A2"><span class="toc-number">1.5.10.</span> <span class="toc-text">数据查询</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%9F%BA%E6%9C%AC%E6%9F%A5%E8%AF%A2"><span class="toc-number">1.5.10.1.</span> <span class="toc-text">基本查询</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#RLIKE%E6%AD%A3%E5%88%99%E5%8C%B9%E9%85%8D"><span class="toc-number">1.5.10.2.</span> <span class="toc-text">RLIKE正则匹配</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#UNION%E8%81%94%E5%90%88"><span class="toc-number">1.5.10.3.</span> <span class="toc-text">UNION联合</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E6%95%B0%E6%8D%AE%E9%87%87%E6%A0%B7"><span class="toc-number">1.5.10.4.</span> <span class="toc-text">数据采样</span></a></li><li class="toc-item toc-level-4"><a class="toc-link" href="#%E8%99%9A%E6%8B%9F%E5%88%97"><span class="toc-number">1.5.10.5.</span> <span class="toc-text">虚拟列</span></a></li></ol></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E5%87%BD%E6%95%B0"><span class="toc-number">1.5.11.</span> <span class="toc-text">函数</span></a><ol class="toc-child"><li class="toc-item toc-level-4"><a class="toc-link" href="#%E5%87%BD%E6%95%B0%E5%88%86%E7%B1%BB"><span class="toc-number">1.5.11.1.</span> <span class="toc-text">函数分类</span></a></li></ol></li></ol></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/2024/03/12/Nmap%E4%BD%BF%E7%94%A8%E6%8C%87%E5%8D%97/" title="Nmap使用指南"><img src="/img/cover/cover_2.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Nmap使用指南"/></a><div class="content"><a class="title" href="/2024/03/12/Nmap%E4%BD%BF%E7%94%A8%E6%8C%87%E5%8D%97/" title="Nmap使用指南">Nmap使用指南</a><time datetime="2024-03-12T02:38:29.000Z" title="发表于 2024-03-12 10:38:29">2024-03-12</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2024/03/10/TCP%E5%8D%8F%E8%AE%AE/" title="TCP协议"><img src="/img/cover/cover_9.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="TCP协议"/></a><div class="content"><a class="title" href="/2024/03/10/TCP%E5%8D%8F%E8%AE%AE/" title="TCP协议">TCP协议</a><time datetime="2024-03-10T08:15:52.000Z" title="发表于 2024-03-10 16:15:52">2024-03-10</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2024/03/06/%E5%B8%B8%E7%94%A8%E7%BD%91%E7%BB%9C%E5%AE%89%E5%85%A8%E8%AE%BE%E5%A4%87/" title="常用网络安全设备"><img src="/img/cover/cover_6.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="常用网络安全设备"/></a><div class="content"><a class="title" href="/2024/03/06/%E5%B8%B8%E7%94%A8%E7%BD%91%E7%BB%9C%E5%AE%89%E5%85%A8%E8%AE%BE%E5%A4%87/" title="常用网络安全设备">常用网络安全设备</a><time datetime="2024-03-06T00:58:05.000Z" title="发表于 2024-03-06 08:58:05">2024-03-06</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2024/03/06/%E6%B8%97%E9%80%8F%E6%B5%8B%E8%AF%95/" title="渗透测试"><img src="/img/cover/cover_9.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="渗透测试"/></a><div class="content"><a class="title" href="/2024/03/06/%E6%B8%97%E9%80%8F%E6%B5%8B%E8%AF%95/" title="渗透测试">渗透测试</a><time datetime="2024-03-06T00:57:01.000Z" title="发表于 2024-03-06 08:57:01">2024-03-06</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2024/03/06/OWASP-TOP-10/" title="OWASP-TOP-10"><img src="/img/cover/cover_2.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="OWASP-TOP-10"/></a><div class="content"><a class="title" href="/2024/03/06/OWASP-TOP-10/" title="OWASP-TOP-10">OWASP-TOP-10</a><time datetime="2024-03-06T00:55:41.000Z" title="发表于 2024-03-06 08:55:41">2024-03-06</time></div></div></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2023 - 2025 By fanhao</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="go-up" type="button" title="回到顶部"><span class="scroll-percent"></span><i class="fas fa-arrow-up"></i></button></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox/fancybox.umd.min.js"></script><div class="js-pjax"></div><script defer="defer" id="ribbon" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/dist/canvas-ribbon.min.js" size="150" alpha="0.6" zIndex="-1" mobile="true" data-click="false"></script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>